Overriding the DataFetcher to return a custom GetRecordsResponseAdapter so that customers can have custom logic to send data to KinesisClientRecord (#1479)

* Make LAM run at configurable interval, tie LeaseDiscoverer with LAM run and tie load balancing with leaseDuration

* remove unwanted SampleApp class

* remove unwanted RecordProcessor and added new parameter class

* Updated comment removed one variable for consistency

* Updated test logic to use Supplier to provide time

* updated logic to count based variance balancing

* Changed variance based balancing to 12

* Changed variance based balancing to 3

* Change logic to balance tied to lease duration

* Change logic to balance tied to LAM run

* Code for backward compatibility

* Code for backward compatibility check

* Code for backward compatibility check

* Best practice to deprecate old constructor

* Best practice to deprecate old constructor

* Best practice to deprecate old constructor

* removed backward compatibility code/constructors

* Formating and remove unused variable

* added formating to avoid build failure

.git-blame-ignore-revs

@@ -0,0 +1 @@
+63ff312818a5f70eab9ec5bf80b53bdd7bf80248

.github/dependabot.yml

@@ -0,0 +1,31 @@
+version: 2
+updates:
+  # branch - master
+  - package-ecosystem: "maven"
+    directory: "/"
+    labels:
+      - "dependencies"
+      - "v3.x"
+    target-branch: "master"
+    schedule:
+      interval: "weekly"
+
+  # branch - v2.x
+  - package-ecosystem: "maven"
+    directory: "/"
+    labels:
+      - "dependencies"
+      - "v2.x"
+    target-branch: "v2.x"
+    schedule:
+      interval: "weekly"
+
+  # branch - v1.x
+  - package-ecosystem: "maven"
+    directory: "/"
+    labels:
+      - "dependencies"
+      - "v1.x"
+    target-branch: "v1.x"
+    schedule:
+      interval: "weekly"

.github/scripts/backwards_compatibility_check.sh

@@ -0,0 +1,144 @@
+#!/bin/bash
+
+TRUE=1
+FALSE=0
+KCL_MAVEN_DIR=~/.m2/repository/software/amazon/kinesis/amazon-kinesis-client
+
+REMOVED_METHODS_FLAG=$FALSE
+LATEST_VERSION=""
+LATEST_JAR=""
+CURRENT_VERSION=""
+CURRENT_JAR=""
+
+# Get the JAR from the latest version release on Maven.
+get_latest_jar() {
+  #  clear the directory so that the latest release will be the only version in the Maven directory after running mvn dependency:get
+  rm -rf "$KCL_MAVEN_DIR"
+  mvn -B dependency:get -Dartifact=software.amazon.kinesis:amazon-kinesis-client:LATEST
+  LATEST_VERSION=$(ls "$KCL_MAVEN_DIR" | grep -E '[0-9]+.[0-9]+.[0-9]+')
+  LATEST_JAR=$KCL_MAVEN_DIR/$LATEST_VERSION/amazon-kinesis-client-$LATEST_VERSION.jar
+}
+
+# Get the JAR with the changes that need to be verified.
+get_current_jar() {
+  mvn -B install -Dmaven.test.skip=true
+  CURRENT_VERSION=$(mvn -q  -Dexec.executable=echo -Dexec.args='${project.version}' --non-recursive exec:exec)
+  CURRENT_JAR=$KCL_MAVEN_DIR/$CURRENT_VERSION/amazon-kinesis-client-$CURRENT_VERSION.jar
+}
+
+is_new_minor_release() {
+  is_new_major_release && return 1
+
+  local latest_minor_version=$(echo "$LATEST_VERSION" | cut -d . -f 2)
+  local current_minor_version=$(echo "$CURRENT_VERSION" | cut -d . -f 2)
+  [[ "$latest_minor_version" != "$current_minor_version" ]]
+  return $?
+}
+
+is_new_major_release() {
+  local latest_major_version=$(echo "$LATEST_VERSION" | cut -d . -f 1)
+  local current_major_version=$(echo "$CURRENT_VERSION" | cut -d . -f 1)
+  [[ "$latest_major_version" != "$current_major_version" ]]
+  return $?
+}
+
+# Skip classes with the KinesisClientInternalApi annotation. These classes are subject to breaking backwards compatibility.
+is_kinesis_client_internal_api() {
+  local current_class="$1"
+  local grep_internal_api_result=$(javap -v -classpath "$LATEST_JAR" "$current_class" | grep KinesisClientInternalApi)
+  [[ "$grep_internal_api_result" != "" ]]
+  return $?
+}
+
+# Skip classes which are not public (e.g. package level). These classes will not break backwards compatibility.
+is_non_public_class() {
+  local current_class="$1"
+  local class_definition=$(javap -classpath "$LATEST_JAR" "$current_class" | head -2 | tail -1)
+  [[ "$class_definition" != *"public"* ]]
+  return $?
+}
+
+# Ignore methods that change from abstract to non-abstract (and vice-versa) if the class is an interface.\
+# Ignore methods that change from synchronized to non-synchronized (and vice-versa)
+ignore_non_breaking_changes() {
+  local current_class="$1"
+  local class_definition=$(javap -classpath "$LATEST_JAR" "$current_class" | head -2 | tail -1)
+  if [[ $class_definition == *"interface"* ]]
+  then
+    LATEST_METHODS=${LATEST_METHODS//abstract /}
+    CURRENT_METHODS=${CURRENT_METHODS//abstract /}
+  else
+    LATEST_METHODS=${LATEST_METHODS//synchronized /}
+    CURRENT_METHODS=${CURRENT_METHODS//synchronized /}
+  fi
+}
+
+# Checks if there are any methods in the latest version that were removed in the current version.
+find_removed_methods() {
+  echo "Checking if methods in current version (v$CURRENT_VERSION) were removed from latest version (v$LATEST_VERSION)"
+  if is_new_minor_release || is_new_major_release
+  then
+    echo "New minor/major release is being performed. Ignoring changes in classes marked with @KinesisClientInternalApi annotation."
+  fi
+  local latest_classes=$(
+    jar tf $LATEST_JAR |
+    grep .class |
+    tr / . |
+    sed 's/\.class$//' |
+    # skip generated proto classes since these have a lot of inherited methods
+    # that are not outputted by javap. besides, generated java code is not a
+    # good indicator of proto compatibility- it will not capture reserved
+    # tags or deprecated fields.
+    grep -v 'software\.amazon\.kinesis\.retrieval\.kpl\.Messages')
+  for class in $latest_classes
+  do
+    if is_kinesis_client_internal_api "$class" || is_non_public_class "$class"
+    then
+      continue
+    fi
+
+    CURRENT_METHODS=$(javap -classpath "$CURRENT_JAR" "$class" 2>/dev/null)
+    if [ -z "$CURRENT_METHODS" ]
+    then
+        echo "Class $class was removed"
+        REMOVED_METHODS_FLAG=$TRUE
+        continue
+    fi
+
+    LATEST_METHODS=$(javap -classpath "$LATEST_JAR" "$class")
+
+    ignore_non_breaking_changes "$class"
+
+    local removed_methods=$(diff <(echo "$LATEST_METHODS") <(echo "$CURRENT_METHODS") | grep '^<')
+
+    # ignore synthetic access methods - these are not available to users and will not break backwards compatibility
+    removed_methods=$(echo "$removed_methods" | grep -v "access\$[0-9]\+")
+
+    if [[ "$removed_methods" != "" ]]
+    then
+      REMOVED_METHODS_FLAG=$TRUE
+      echo "$class does not have method(s):"
+      echo "$removed_methods"
+    fi
+  done
+}
+
+get_backwards_compatible_result() {
+  if [[ $REMOVED_METHODS_FLAG == $TRUE ]]
+  then
+    echo "Current KCL version $CURRENT_VERSION is not backwards compatible with version $LATEST_VERSION. See output above for removed packages/methods."
+    is_new_major_release || exit 1
+  else
+    echo "Current KCL version $CURRENT_VERSION is backwards compatible with version $LATEST_VERSION."
+    exit 0
+  fi
+}
+
+main() {
+  get_latest_jar
+  get_current_jar
+  find_removed_methods
+  get_backwards_compatible_result
+}
+
+main

.github/workflows/maven.yml

@@ -0,0 +1,65 @@
+# This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-maven
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: Java CI with Maven
+on:
+  push:
+    branches:
+      - "master"
+      - "v2.x"
+      - "v1.x"
+  pull_request:
+    branches:
+      - "master"
+      - "v2.x"
+      - "v1.x"
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK 8
+        uses: actions/setup-java@v4
+        with:
+          java-version: '8'
+          distribution: 'corretto'
+      - name: Build with Maven
+        run: mvn -B package --file pom.xml -DskipITs
+  backwards-compatible-check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK 8
+        uses: actions/setup-java@v4
+        with:
+          java-version: '8'
+          distribution: 'corretto'
+      - name: Check backwards compatibility of changes
+        run: .github/scripts/backwards_compatibility_check.sh
+  auto-merge:
+    needs: [build]
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.user.login == 'dependabot[bot]'
+    steps:
+      - name: Dependabot metadata
+        id: metadata
+        uses: dependabot/fetch-metadata@v2
+        with:
+          alert-lookup: true
+          github-token: "${{ secrets.GITHUB_TOKEN }}"
+      - name: Enable auto-merge for Dependabot PRs
+        if: steps.metadata.outputs.update-type == 'version-update:semver-patch' && steps.metadata.outputs.cvss > 0
+        run: gh pr merge --auto --merge "$PR_URL"
+        env:
+          PR_URL: ${{github.event.pull_request.html_url}}
+          GH_TOKEN: ${{secrets.GITHUB_TOKEN}}

.gitignore

@@ -1,3 +1,6 @@
 target/
 AwsCredentials.properties
 .idea
+*.iml
+*.swp
+.DS_Store

CHANGELOG.md

@@ -1,306 +1,64 @@
 # Changelog
 
-## Release 1.9.1 (April 30, 2018)
-* Added the ability to create a prepared checkpoint when at `SHARD_END`.
-  * [PR #301](https://github.com/awslabs/amazon-kinesis-client/pull/301)
-* Added the ability to subscribe to worker state change events.  
-  * [PR #291](https://github.com/awslabs/amazon-kinesis-client/pull/291)
-* Added support for custom lease managers.  
-  A custom `LeaseManager` can be provided to `Worker.Builder` that will be used to provide lease services. 
-  This makes it possible to implement custom lease management systems in addition to the default DynamoDB system.  
-  * [PR #297](https://github.com/awslabs/amazon-kinesis-client/pull/297)
-* Updated the version of the AWS Java SDK to 1.11.219
+For **1.x** release notes, please see [v1.x/CHANGELOG.md](https://github.com/awslabs/amazon-kinesis-client/blob/v1.x/CHANGELOG.md)
 
-## Release 1.9.0 (February 6, 2018)
-* Introducing support for ListShards API. This API is used in place of DescribeStream API to provide more throughput during ShardSyncTask. Please consult the [AWS Documentation for ListShards](https://docs.aws.amazon.com/kinesis/latest/APIReference/API_ListShards.html) for more information.
-  * ListShards supports higher call rate, which should reduce instances of throttling when attempting to synchronize the shard list.
-  * __WARNING: `ListShards` is a new API, and may require updating any explicit IAM policies__
-  * Added configuration parameters for ListShards usage
-  
-  | Name | Default | Description |
-  | ---- | ------- | ----------- |
-  | [listShardsBackoffTimeInMillis](https://github.com/awslabs/amazon-kinesis-client/blob/3ae916c5fcdccd6b835c86ba7f6f53dd5b4c8b04/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java#L1412) | 1500 ms | This is the default backoff time between 2 ListShards calls when throttled. |
-  | [listShardsRetryAttempts](https://github.com/awslabs/amazon-kinesis-client/blob/3ae916c5fcdccd6b835c86ba7f6f53dd5b4c8b04/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java#L1423) | 50 | This is the maximum number of times the KinesisProxy will retry to make ListShards calls on being throttled. |
-  
-* Updating the version of AWS Java SDK to 1.11.272.
-  * Version 1.11.272 is now the minimum support version of the SDK.
-* Deprecating the following methods, and classes. These methods, and classes will be removed in a future release.
-  * Deprecated [IKinesisProxy#getStreamInfo](https://github.com/awslabs/amazon-kinesis-client/blob/3ae916c5fcdccd6b835c86ba7f6f53dd5b4c8b04/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/IKinesisProxy.java#L48-L62).
-  * Deprecated [IKinesisProxyFactory](https://github.com/awslabs/amazon-kinesis-client/blob/3ae916c5fcdccd6b835c86ba7f6f53dd5b4c8b04/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/IKinesisProxyFactory.java).
-  * Deprecated [KinesisProxyFactory](https://github.com/awslabs/amazon-kinesis-client/blob/3ae916c5fcdccd6b835c86ba7f6f53dd5b4c8b04/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxyFactory.java).
-  * Deprecated certain [KinesisProxy](https://github.com/awslabs/amazon-kinesis-client/blob/3ae916c5fcdccd6b835c86ba7f6f53dd5b4c8b04/src/main/java/com/amazonaws/services/kinesis/clientlibrary/proxies/KinesisProxy.java) constructors.
-    * [PR #293](https://github.com/awslabs/amazon-kinesis-client/pull/293)
+For **2.x** release notes, please see [v2.x/CHANGELOG.md](https://github.com/awslabs/amazon-kinesis-client/blob/v2.x/CHANGELOG.md)
 
-## Release 1.8.10
-* Allow providing a custom IKinesisProxy implementation.
-  * [PR #274](https://github.com/awslabs/amazon-kinesis-client/pull/274)
-* Checkpointing on a different thread should no longer emit a warning about NullMetricsScope.
-  * [PR #284](https://github.com/awslabs/amazon-kinesis-client/pull/284)
-  * [Issue #48](https://github.com/awslabs/amazon-kinesis-client/issues/48)
-* Upgraded the AWS Java SDK to version 1.11.271
-  * [PR #287](https://github.com/awslabs/amazon-kinesis-client/pull/287)
+---
+### Release 3.0.3 (May 7, 2025)
+* [#1464](https://github.com/awslabs/amazon-kinesis-client/pull/1464) Add config for LeaseAssignmentManager frequency and improve assignment time of newly created leases
+* [#1463](https://github.com/awslabs/amazon-kinesis-client/pull/1463) Extend ShardConsumer constructor to have ConsumerTaskFactory as a parameter to support [DynamoDB Streams Kinesis Adapter](https://github.com/awslabs/dynamodb-streams-kinesis-adapter) compatibility
+* [#1472](https://github.com/awslabs/amazon-kinesis-client/pull/1472) Remove unused synchronized keyword
 
-## Release 1.8.9
-* Allow disabling check for the case where a child shard has an open parent shard.  
-  There is a race condition where it's possible for the a parent shard to appear open, while having child shards. This check can now be disabled by setting [`ignoreUnexpectedChildShards`](https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java#L1037) to true.
-  * [PR #240](https://github.com/awslabs/amazon-kinesis-client/pull/240)
-  * [Issue #210](https://github.com/awslabs/amazon-kinesis-client/issues/210)
-* Upgraded the AWS SDK for Java to 1.11.261
-  * [PR #281](https://github.com/awslabs/amazon-kinesis-client/pull/281)
+### Release 3.0.2 (March 12, 2025)
+* [#1443](https://github.com/awslabs/amazon-kinesis-client/pull/1443) Reduce DynamoDB IOPS for smaller stream and worker count applications
+* The below two PRs are intended to support [DynamoDB Streams Kinesis Adapter](https://github.com/awslabs/dynamodb-streams-kinesis-adapter) compatibility
+  * [#1441](https://github.com/awslabs/amazon-kinesis-client/pull/1441) Make consumerTaskFactory overridable by customers
+  * [#1440](https://github.com/awslabs/amazon-kinesis-client/pull/1440) Make ShutdownTask, ProcessTask, InitializeTask, BlockOnParentTask, and ShutdownNotificationTask overridable by customers
+* [#1437](https://github.com/awslabs/amazon-kinesis-client/pull/1437) Suppress LeaseAssignmentManager excessive WARN logs when there is no issue
+* [#1439](https://github.com/awslabs/amazon-kinesis-client/pull/1439) Upgrade io.netty:netty-handler from 4.1.108.Final to 4.1.118.Final
+* [#1400](https://github.com/awslabs/amazon-kinesis-client/pull/1400) Upgrade com.fasterxml.jackson.core:jackson-databind from 2.10.1 to 2.12.7.1
 
-## Release 1.8.8
-* Fixed issues with leases losses due to `ExpiredIteratorException` in `PrefetchGetRecordsCache` and `AsynchronousFetchingStrategy`.  
-  PrefetchGetRecordsCache will request for a new iterator and start fetching data again.  
-  * [PR#263](https://github.com/awslabs/amazon-kinesis-client/pull/263)
-* Added warning message for long running tasks.  
-  Logging long running tasks can be enabled by setting the following configuration property:
-  
-  | Name | Default | Description |
-  | ---- | ------- | ----------- |
-  | [`logWarningForTaskAfterMillis`](https://github.com/awslabs/amazon-kinesis-client/blob/3de901ea9327370ed732af86c4d4999c8d99541c/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java#L1367) | Not set | Milliseconds after which the logger will log a warning message for the long running task |
-  
-  * [PR#259](https://github.com/awslabs/amazon-kinesis-client/pull/259)
-* Handling spurious lease renewal failures gracefully.  
-  Added better handling of DynamoDB failures when updating leases.  These failures would occur when a request to DynamoDB appeared to fail, but was actually successful.  
-  * [PR#247](https://github.com/awslabs/amazon-kinesis-client/pull/247)
-* ShutdownTask gets retried if the previous attempt on the ShutdownTask fails.
-  * [PR#267](https://github.com/awslabs/amazon-kinesis-client/pull/267)
-* Fix for using maxRecords from `KinesisClientLibConfiguration` in `GetRecordsCache` for fetching records.
-  * [PR#264](https://github.com/awslabs/amazon-kinesis-client/pull/264)
+### Release 3.0.1 (November 14, 2024)
+* [#1401](https://github.com/awslabs/amazon-kinesis-client/pull/1401) Fixed the lease graceful handoff behavior in the multi-stream processing mode
+* [#1398](https://github.com/awslabs/amazon-kinesis-client/pull/1398) Addressed several KCL 3.0 related issues raised via GitHub
+    * Fixed transitive dependencies and added a Maven plugin to catch potential transitive dependency issues at build time
+    * Removed the redundant shutdown of the leaseCoordinatorThreadPool
+    * Fixed typo THROUGHOUT_PUT_KBPS
+    * Fixed issues in scheduler shutdown sequence
 
-## Release 1.8.7
-* Don't add a delay for synchronous requests to Kinesis  
-  Removes a delay that had been added for synchronous `GetRecords` calls to Kinesis. 
-  * [PR #256](https://github.com/awslabs/amazon-kinesis-client/pull/256)
+* Note: If you are using [multi-stream processing with KCL](https://docs.aws.amazon.com/streams/latest/dev/kcl-multi-stream.html), you need to use the release 3.0.1 or later.
 
-## Release 1.8.6
-* Add prefetching of records from Kinesis  
-  Prefetching will retrieve and queue additional records from Kinesis while the application is processing existing records.  
-  Prefetching can be enabled by setting [`dataFetchingStrategy`](https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java#L1317) to `PREFETCH_CACHED`. Once enabled an additional fetching thread will be started to retrieve records from Kinesis. Retrieved records will be held in a queue until the application is ready to process them.  
-  Pre-fetching supports the following configuration values:  
-  
-  | Name | Default | Description |
-  | ---- | ------- | ----------- |
-  | [`dataFetchingStrategy`](https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java#L1317) | `DEFAULT` | Which data fetching strategy to use |
-  | [`maxPendingProcessRecordsInput`](https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java#L1296) | 3 | The maximum number of process records input that can be queued |
-  | [`maxCacheByteSize`](https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java#L1307) | 8 MiB | The maximum number of bytes that can be queued |
-  | [`maxRecordsCount`](https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java#L1326) | 30,000 | The maximum number of records that can be queued |
-  | [`idleMillisBetweenCalls`](https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java#L1353) | 1,500 ms | The amount of time to wait between calls to Kinesis |
-  
-  * [PR #246](https://github.com/awslabs/amazon-kinesis-client/pull/246)
-
-## Release 1.8.5 (September 26, 2017)
-* Only advance the shard iterator for the accepted response.  
-  This fixes a race condition in the `KinesisDataFetcher` when it's being used to make asynchronous requests.  The shard iterator is now only advanced when the retriever calls `DataFetcherResult#accept()`.
-  * [PR #230](https://github.com/awslabs/amazon-kinesis-client/pull/230)
-  * [Issue #231](https://github.com/awslabs/amazon-kinesis-client/issues/231)
-
-## Release 1.8.4 (September 22, 2017)
-* Create a new completion service for each request.  
-  This ensures that canceled tasks are discarded.  This will prevent a cancellation exception causing issues processing records.
-  * [PR #227](https://github.com/awslabs/amazon-kinesis-client/pull/227)
-  * [Issue #226](https://github.com/awslabs/amazon-kinesis-client/issues/226)
-
-## Release 1.8.3 (September 22, 2017)
-* Call shutdown on the retriever when the record processor is being shutdown  
-  This fixes a bug that could leak threads if using the [`AsynchronousGetRecordsRetrievalStrategy`](https://github.com/awslabs/amazon-kinesis-client/blob/9a82b6bd05b3c9c5f8581af007141fa6d5f0fc4e/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/AsynchronousGetRecordsRetrievalStrategy.java#L42) is being used.  
-  The asynchronous retriever is only used when [`KinesisClientLibConfiguration#retryGetRecordsInSeconds`](https://github.com/awslabs/amazon-kinesis-client/blob/01d2688bc6e68fd3fe5cb698cb65299d67ac930d/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java#L227), and [`KinesisClientLibConfiguration#maxGetRecordsThreadPool`](https://github.com/awslabs/amazon-kinesis-client/blob/01d2688bc6e68fd3fe5cb698cb65299d67ac930d/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java#L230) are set.
-  * [PR #222](https://github.com/awslabs/amazon-kinesis-client/pull/222)
-
-## Release 1.8.2 (September 20, 2017)
-* Add support for two phase checkpoints  
-  Applications can now set a pending checkpoint, before completing the checkpoint operation. Once the application has completed its checkpoint steps, the final checkpoint will clear the pending checkpoint.  
-  Should the checkpoint fail the attempted sequence number is provided in the [`InitializationInput#getPendingCheckpointSequenceNumber`](https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/clientlibrary/types/InitializationInput.java#L81) otherwise the value will be null.
-  * [PR #188](https://github.com/awslabs/amazon-kinesis-client/pull/188)
-* Support timeouts, and retry for GetRecords calls.  
-  Applications can now set timeouts for GetRecord calls to Kinesis.  As part of setting the timeout, the application must also provide a thread pool size for concurrent requests.
-  * [PR #214](https://github.com/awslabs/amazon-kinesis-client/pull/214)
-* Notification when the lease table is throttled  
-  When writes, or reads, to the lease table are throttled a warning will be emitted.  If you're seeing this warning you should increase the IOPs for your lease table to prevent processing delays.
-  * [PR #212](https://github.com/awslabs/amazon-kinesis-client/pull/212)
-* Support configuring the graceful shutdown timeout for MultiLang Clients  
-  This adds support for setting the timeout that the Java process will wait for the MutliLang client to complete graceful shutdown.  The timeout can be configured by adding `shutdownGraceMillis` to the properties file set to the number of milliseconds to wait.
-  * [PR #204](https://github.com/awslabs/amazon-kinesis-client/pull/204)
-
-## Release 1.8.1 (August 2, 2017)
-* Support timeouts for calls to the MultiLang Daemon
-  This adds support for setting a timeout when dispatching records to the client record processor. If the record processor doesn't respond within the timeout the parent Java process will be terminated. This is a temporary fix to handle cases where the KCL becomes blocked while waiting for a client record processor.
-  The timeout for the this can be set by adding `timeoutInSeconds = <timeout value>`. The default for this is no timeout.  
-  __Setting this can cause the KCL to exit suddenly, before using this ensure that you have an automated restart for your application__
-  * [PR #195](https://github.com/awslabs/amazon-kinesis-client/pull/195)
-  * [Issue #185](https://github.com/awslabs/amazon-kinesis-client/issues/185)
-
-## Release 1.8.0 (July 25, 2017)
-* Execute graceful shutdown on its own thread
-  * [PR #191](https://github.com/awslabs/amazon-kinesis-client/pull/191)
-  * [Issue #167](https://github.com/awslabs/amazon-kinesis-client/issues/167)
-* Added support for controlling the size of the lease renewer thread pool
-  * [PR #177](https://github.com/awslabs/amazon-kinesis-client/pull/177)
-  * [Issue #171](https://github.com/awslabs/amazon-kinesis-client/issues/171)
-* Require Java 8 and later  
-  __Java 8 is now required for versions 1.8.0 of the amazon-kinesis-client and later.__
-  * [PR #176](https://github.com/awslabs/amazon-kinesis-client/issues/176)
-
-## Release 1.7.6 (June 21, 2017)
-* Added support for graceful shutdown in MultiLang Clients
-  * [PR #174](https://github.com/awslabs/amazon-kinesis-client/pull/174)
-  * [PR #182](https://github.com/awslabs/amazon-kinesis-client/pull/182)
-* Updated documentation for `v2.IRecordProcessor#shutdown`, and `KinesisClientLibConfiguration#idleTimeBetweenReadsMillis`
-  * [PR #170](https://github.com/awslabs/amazon-kinesis-client/pull/170)
-* Updated to version 1.11.151 of the AWS Java SDK
-  * [PR #183](https://github.com/awslabs/amazon-kinesis-client/pull/183)
-
-## Release 1.7.5 (April 7, 2017)
-* Correctly handle throttling for DescribeStream, and save accumulated progress from individual calls.
-  * [PR #152](https://github.com/awslabs/amazon-kinesis-client/pull/152)
-* Upgrade to version 1.11.115 of the AWS Java SDK
-  * [PR #155](https://github.com/awslabs/amazon-kinesis-client/pull/155)
-  
-## Release 1.7.4 (February 27, 2017)
-* Fixed an issue building JavaDoc for Java 8.
-  * [Issue #18](https://github.com/awslabs/amazon-kinesis-client/issues/18)
-  * [PR #141](https://github.com/awslabs/amazon-kinesis-client/pull/141)
-* Reduce Throttling Messages to WARN, unless throttling occurs 6 times consecutively.
-  * [Issue #4](https://github.com/awslabs/amazon-kinesis-client/issues/4)
-  * [PR #140](https://github.com/awslabs/amazon-kinesis-client/pull/140)
-* Fixed two bugs occurring in requestShutdown.
-  * Fixed a bug that prevented the worker from shutting down, via requestShutdown, when no leases were held.
-    * [Issue #128](https://github.com/awslabs/amazon-kinesis-client/issues/128)
-  * Fixed a bug that could trigger a NullPointerException if leases changed during requestShutdown.
-    * [Issue #129](https://github.com/awslabs/amazon-kinesis-client/issues/129)
-  * [PR #139](https://github.com/awslabs/amazon-kinesis-client/pull/139)
-* Upgraded the AWS SDK Version to 1.11.91
-  * [PR #138](https://github.com/awslabs/amazon-kinesis-client/pull/138)
-* Use an executor returned from `ExecutorService.newFixedThreadPool` instead of constructing it by hand.
-  * [PR #135](https://github.com/awslabs/amazon-kinesis-client/pull/135)
-* Correctly initialize DynamoDB client, when endpoint is explicitly set.
-  * [PR #142](https://github.com/awslabs/amazon-kinesis-client/pull/142)
-
-## Release 1.7.3 (January 9, 2017)
-* Upgrade to the newest AWS Java SDK.
-  * [Amazon Kinesis Client Issue #27](https://github.com/awslabs/amazon-kinesis-client-python/issues/27)
-  * [PR #126](https://github.com/awslabs/amazon-kinesis-client/pull/126)
-  * [PR #125](https://github.com/awslabs/amazon-kinesis-client/pull/125)
-* Added a direct dependency on commons-logging.
-  * [Issue #123](https://github.com/awslabs/amazon-kinesis-client/issues/123)
-  * [PR #124](https://github.com/awslabs/amazon-kinesis-client/pull/124)
-* Make ShardInfo public to allow for custom ShardPrioritization strategies.
-  * [Issue #120](https://github.com/awslabs/amazon-kinesis-client/issues/120)
-  * [PR #127](https://github.com/awslabs/amazon-kinesis-client/pull/127)
-
-## Release 1.7.2 (November 7, 2016)
-* MultiLangDaemon Feature Updates
-  The MultiLangDaemon has been upgraded to use the v2 interfaces, which allows access to enhanced checkpointing, and more information during record processor initialization. The MultiLangDaemon clients must be updated before they can take advantage of these new features.
-  
-## Release 1.7.1 (November 3, 2016)
-* General
-  * Allow disabling shard synchronization at startup.
-    * Applications can disable shard synchronization at startup.  Disabling shard synchronization can application startup times for very large streams.
-    * [PR #102](https://github.com/awslabs/amazon-kinesis-client/pull/102)
-  * Applications can now request a graceful shutdown, and record processors that implement the IShutdownNotificationAware will be given a chance to checkpoint before being shutdown.
-    * This adds a [new interface](https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/clientlibrary/interfaces/v2/IShutdownNotificationAware.java), and a [new method on Worker](https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/Worker.java#L539).
-    * [PR #109](https://github.com/awslabs/amazon-kinesis-client/pull/109)
-    * Solves [Issue #79](https://github.com/awslabs/amazon-kinesis-client/issues/79)
-* MultiLangDaemon
-  * Applications can now use credential provides that accept string parameters.
-    * [PR #99](https://github.com/awslabs/amazon-kinesis-client/pull/99)
-  * Applications can now use different credentials for each service.
-    * [PR #111](https://github.com/awslabs/amazon-kinesis-client/pull/111)
-
-## Release 1.7.0 (August 22, 2016)
-* Add support for time based iterators ([See GetShardIterator Documentation](http://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetShardIterator.html))
-  * [PR #94](https://github.com/awslabs/amazon-kinesis-client/pull/94)
-  The `KinesisClientLibConfiguration` now supports providing an initial time stamp position.
-  * This position is only used if there is no current checkpoint for the shard.
-  * This setting cannot be used with DynamoDB Streams
-  Resolves [Issue #88](https://github.com/awslabs/amazon-kinesis-client/issues/88)
-* Allow Prioritization of Parent Shards for Task Assignment
-  * [PR #95](https://github.com/awslabs/amazon-kinesis-client/pull/95)
-  The `KinesisClientLibconfiguration` now supports providing a `ShardPrioritization` strategy.  This strategy controls how the `Worker` determines which `ShardConsumer` to call next.  This can improve processing for streams that split often, such as DynamoDB Streams.
-* Remove direct dependency on `aws-java-sdk-core`, to allow independent versioning.
-  * [PR #92](https://github.com/awslabs/amazon-kinesis-client/pull/92)
-  **You may need to add a direct dependency on aws-java-sdk-core if other dependencies include an older version.**
-
-## Release 1.6.5 (July 25, 2016)
-* Change LeaseManager to call DescribeTable before attempting to create the lease table.
-  * [Issue #36](https://github.com/awslabs/amazon-kinesis-client/issues/36)
-  * [PR #41](https://github.com/awslabs/amazon-kinesis-client/pull/41)
-  * [PR #67](https://github.com/awslabs/amazon-kinesis-client/pull/67)
-* Allow DynamoDB lease table name to be specified
-  * [PR #61](https://github.com/awslabs/amazon-kinesis-client/pull/61)
-* Add approximateArrivalTimestamp for JsonFriendlyRecord
-  * [PR #86](https://github.com/awslabs/amazon-kinesis-client/pull/86)
-* Shutdown lease renewal thread pool on exit.
-  * [PR #84](https://github.com/awslabs/amazon-kinesis-client/pull/84)
-* Wait for CloudWatch publishing thread to finish before exiting.
-  * [PR #82](https://github.com/awslabs/amazon-kinesis-client/pull/82)
-* Added unit, and integration tests for the library.
-
-## Release 1.6.4 (July 6, 2016)
-* Upgrade to AWS SDK for Java 1.11.14
-  * [Issue #74](https://github.com/awslabs/amazon-kinesis-client/issues/74)
-  * [Issue #73](https://github.com/awslabs/amazon-kinesis-client/issues/73)
-* **Maven Artifact Signing Change** 
-  * Artifacts are now signed by the identity `Amazon Kinesis Tools <amazon-kinesis-tools@amazon.com>`
-
-## Release 1.6.3 (May 12, 2016)
-* Fix format exception caused by DEBUG log in LeaseTaker [Issue # 68](https://github.com/awslabs/amazon-kinesis-client/issues/68)
-
-## Release 1.6.2 (March 23, 2016)
-* Support for specifying max leases per worker and max leases to steal at a time.
-* Support for specifying initial DynamoDB table read and write capacity.
-* Support for parallel lease renewal.
-* Support for graceful worker shutdown.
-* Change DefaultCWMetricsPublisher log level to debug. [PR # 49](https://github.com/awslabs/amazon-kinesis-client/pull/49)
-* Avoid NPE in MLD record processor shutdown if record processor was not initialized. [Issue # 29](https://github.com/awslabs/amazon-kinesis-client/issues/29)
-
-## Release 1.6.1 (September 23, 2015)
-* Expose [approximateArrivalTimestamp](http://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetRecords.html) for Records in processRecords API call.
-
-## Release 1.6.0 (July 31, 2015)
-* Restores compatibility with [dynamodb-streams-kinesis-adapter](https://github.com/awslabs/dynamodb-streams-kinesis-adapter) (which was broken in 1.4.0).
-
-## Release 1.5.1 (July 20, 2015)
-* KCL maven artifact 1.5.0 does not work with JDK 7. This release addresses this issue.
-
-## Release 1.5.0 (July 9, 2015)
-* **[Metrics Enhancements][kinesis-guide-monitoring-with-kcl]**
-	* Support metrics level and dimension configurations to control CloudWatch metrics emitted by the KCL.
-	* Add new metrics that track time spent in record processor methods.
-	* Disable WorkerIdentifier dimension by default.
-* **Exception Reporting** &mdash; Do not silently ignore exceptions in ShardConsumer.
-* **AWS SDK Component Dependencies** &mdash; Depend only on AWS SDK components that are used.
-
-## Release 1.4.0 (June 2, 2015)
-* Integration with the **[Kinesis Producer Library (KPL)][kinesis-guide-kpl]**
-	* Automatically de-aggregate records put into the Kinesis stream using the KPL.
-	* Support checkpointing at the individual user record level when multiple user records are aggregated into one Kinesis record using the KPL.
-
- See [Consumer De-aggregation with the KCL][kinesis-guide-consumer-deaggregation] for details.
-
-## Release 1.3.0 (May 22, 2015)
-* A new metric called "MillisBehindLatest", which tracks how far consumers are from real time, is now uploaded to CloudWatch.
-
-## Release 1.2.1 (January 26, 2015)
-* **MultiLangDaemon** &mdash; Changes to the MultiLangDaemon to make it easier to provide a custom worker.
-
-## Release 1.2 (October 21, 2014)
-* **Multi-Language Support** &mdash; Amazon KCL now supports implementing record processors in any language by communicating with the daemon over [STDIN and STDOUT][multi-lang-protocol]. Python developers can directly use the [Amazon Kinesis Client Library for Python][kclpy] to write their data processing applications.
-
-## Release 1.1 (June 30, 2014)
-* **Checkpointing at a specific sequence number** &mdash; The IRecordProcessorCheckpointer interface now supports checkpointing at a sequence number specified by the record processor.
-* **Set region** &mdash; KinesisClientLibConfiguration now supports setting the region name to indicate the location of the Amazon Kinesis service. The Amazon DynamoDB table and Amazon CloudWatch metrics associated with your application will also use this region setting.
-
-[kinesis]: http://aws.amazon.com/kinesis
-[kinesis-forum]: http://developer.amazonwebservices.com/connect/forum.jspa?forumID=169
-[kinesis-client-library-issues]: https://github.com/awslabs/amazon-kinesis-client/issues
-[docs-signup]: http://docs.aws.amazon.com/AWSSdkDocsJava/latest/DeveloperGuide/java-dg-setup.html
-[kinesis-guide]: http://docs.aws.amazon.com/kinesis/latest/dev/introduction.html
-[kinesis-guide-begin]: http://docs.aws.amazon.com/kinesis/latest/dev/before-you-begin.html
-[kinesis-guide-create]: http://docs.aws.amazon.com/kinesis/latest/dev/step-one-create-stream.html
-[kinesis-guide-applications]: http://docs.aws.amazon.com/kinesis/latest/dev/kinesis-record-processor-app.html
-[kinesis-guide-monitoring-with-kcl]: http://docs.aws.amazon.com//kinesis/latest/dev/monitoring-with-kcl.html
-[kinesis-guide-kpl]: http://docs.aws.amazon.com//kinesis/latest/dev/developing-producers-with-kpl.html
-[kinesis-guide-consumer-deaggregation]: http://docs.aws.amazon.com//kinesis/latest/dev/kinesis-kpl-consumer-deaggregation.html
-[kclpy]: https://github.com/awslabs/amazon-kinesis-client-python
-[multi-lang-protocol]: https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/multilang/package-info.java
+### Release 3.0.0 (November 06, 2024)
+* New lease assignment / load balancing algorithm
+    * KCL 3.x introduces a new lease assignment and load balancing algorithm. It assigns leases among workers based on worker utilization metrics and throughput on each lease, replacing the previous lease count-based lease assignment algorithm.
+    * When KCL detects higher variance in CPU utilization among workers, it proactively reassigns leases from over-utilized workers to under-utilized workers for even load balancing. This ensures even CPU utilization across workers and removes the need to over-provision the stream processing compute hosts.
+* Optimized DynamoDB RCU usage
+    * KCL 3.x optimizes DynamoDB read capacity unit (RCU) usage on the lease table by implementing a global secondary index with leaseOwner as the partition key. This index mirrors the leaseKey attribute from the base lease table, allowing workers to efficiently discover their assigned leases by querying the index instead of scanning the entire table.
+    * This approach significantly reduces read operations compared to earlier KCL versions, where workers performed full table scans, resulting in higher RCU consumption.
+* Graceful lease handoff
+    * KCL 3.x introduces a feature called "graceful lease handoff" to minimize data reprocessing during lease reassignments. Graceful lease handoff allows the current worker to complete checkpointing of processed records before transferring the lease to another worker. For graceful lease handoff, you should implement checkpointing logic within the existing `shutdownRequested()` method.
+    * This feature is enabled by default in KCL 3.x, but you can turn off this feature by adjusting the configuration property `isGracefulLeaseHandoffEnabled`.
+    * While this approach significantly reduces the probability of data reprocessing during lease transfers, it doesn't completely eliminate the possibility. To maintain data integrity and consistency, it's crucial to design your downstream consumer applications to be idempotent. This ensures that the application can handle potential duplicate record processing without adverse effects.
+* New DynamoDB metadata management artifacts
+    * KCL 3.x introduces two new DynamoDB tables for improved lease management:
+        * Worker metrics table: Records CPU utilization metrics from each worker. KCL uses these metrics for optimal lease assignments, balancing resource utilization across workers. If CPU utilization metric is not available, KCL assigns leases to balance the total sum of shard throughput per worker instead.
+        * Coordinator state table: Stores internal state information for workers. Used to coordinate in-place migration from KCL 2.x to KCL 3.x and leader election among workers.
+    * Follow this [documentation](https://docs.aws.amazon.com/streams/latest/dev/kcl-migration-from-2-3.html#kcl-migration-from-2-3-IAM-permissions) to add required IAM permissions for your KCL application.
+* Other improvements and changes
+    * Dependency on the AWS SDK for Java 1.x has been fully removed.
+        * The Glue Schema Registry integration functionality no longer depends on AWS SDK for Java 1.x. Previously, it required this as a transient dependency.
+        * Multilangdaemon has been upgraded to use AWS SDK for Java 2.x. It no longer depends on AWS SDK for Java 1.x.
+    * `idleTimeBetweenReadsInMillis` (PollingConfig) now has a minimum default value of 200.
+        * This polling configuration property determines the [publishers](https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/retrieval/polling/PrefetchRecordsPublisher.java) wait time between GetRecords calls in both success and failure cases. Previously, setting this value below 200 caused unnecessary throttling. This is because Amazon Kinesis Data Streams supports up to five read transactions per second per shard for shared-throughput consumers.
+    * Shard lifecycle management is improved to deal with edge cases around shard splits and merges to ensure records continue being processed as expected.
+* Migration
+    * The programming interfaces of KCL 3.x remain identical with KCL 2.x for an easier migration, with the exception of those applications that do not use the recommended approach of using the Config Builder. These applications will have to refer to [the troubleshooting guide](https://docs.aws.amazon.com/streams/latest/dev/troubleshooting-consumers.html#compiliation-error-leasemanagementconfig). For detailed migration instructions, please refer to the [Migrate consumers from KCL 2.x to KCL 3.x](https://docs.aws.amazon.com/streams/latest/dev/kcl-migration-from-2-3.html) page in the Amazon Kinesis Data Streams developer guide.
+* Configuration properties
+    * New configuration properties introduced in KCL 3.x are listed in this [doc](https://github.com/awslabs/amazon-kinesis-client/blob/master/docs/kcl-configurations.md#new-configurations-in-kcl-3x).
+    * Deprecated configuration properties in KCL 3.x are listed in this [doc](https://github.com/awslabs/amazon-kinesis-client/blob/master/docs/kcl-configurations.md#discontinued-configuration-properties-in-kcl-3x). You need to keep the deprecated configuration properties during the migration from any previous KCL version to KCL 3.x.
+* Metrics
+    * New CloudWatch metrics introduced in KCL 3.x are explained in the [Monitor the Kinesis Client Library with Amazon CloudWatch](https://docs.aws.amazon.com/streams/latest/dev/monitoring-with-kcl.html) in the Amazon Kinesis Data Streams developer guide. The following operations are newly added in KCL 3.x:
+        * `LeaseAssignmentManager`
+        * `WorkerMetricStatsReporter`
+        * `LeaseDiscovery`

LICENSE.txt

@@ -1,40 +1,201 @@
+                                Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
 
-Amazon Software License
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
 
-This Amazon Software License (“License”) governs your use, reproduction, and distribution of the accompanying software as specified below.
-1. Definitions
+   1. Definitions.
 
-“Licensor” means any person or entity that distributes its Work.
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
 
-“Software” means the original work of authorship made available under this License.
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
 
-“Work” means the Software and any additions to or derivative works of the Software that are made available under this License.
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
 
-The terms “reproduce,” “reproduction,” “derivative works,” and “distribution” have the meaning as provided under U.S. copyright law; provided, however, that for the purposes of this License, derivative works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work.
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
 
-Works, including the Software, are “made available” under this License by including in or with the Work either (a) a copyright notice referencing the applicability of this License to the Work, or (b) a copy of this License.
-2. License Grants
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
 
-2.1 Copyright Grant. Subject to the terms and conditions of this License, each Licensor grants to you a perpetual, worldwide, non-exclusive, royalty-free, copyright license to reproduce, prepare derivative works of, publicly display, publicly perform, sublicense and distribute its Work and any resulting derivative works in any form.
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
 
-2.2 Patent Grant. Subject to the terms and conditions of this License, each Licensor grants to you a perpetual, worldwide, non-exclusive, royalty-free patent license to make, have made, use, sell, offer for sale, import, and otherwise transfer its Work, in whole or in part. The foregoing license applies only to the patent claims licensable by Licensor that would be infringed by Licensor’s Work (or portion thereof) individually and excluding any combinations with any other materials or technology.
-3. Limitations
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
 
-3.1 Redistribution. You may reproduce or distribute the Work only if (a) you do so under this License, (b) you include a complete copy of this License with your distribution, and (c) you retain without modification any copyright, patent, trademark, or attribution notices that are present in the Work.
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
 
-3.2 Derivative Works. You may specify that additional or different terms apply to the use, reproduction, and distribution of your derivative works of the Work (“Your Terms”) only if (a) Your Terms provide that the use limitation in Section 3.3 applies to your derivative works, and (b) you identify the specific derivative works that are subject to Your Terms. Notwithstanding Your Terms, this License (including the redistribution requirements in Section 3.1) will continue to apply to the Work itself.
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
 
-3.3 Use Limitation. The Work and any derivative works thereof only may be used or intended for use with the web services, computing platforms or applications provided by Amazon.com, Inc. or its affiliates, including Amazon Web Services, Inc.
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
 
-3.4 Patent Claims. If you bring or threaten to bring a patent claim against any Licensor (including any claim, cross-claim or counterclaim in a lawsuit) to enforce any patents that you allege are infringed by any Work, then your rights under this License from such Licensor (including the grants in Sections 2.1 and 2.2) will terminate immediately.
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
 
-3.5 Trademarks. This License does not grant any rights to use any Licensor’s or its affiliates’ names, logos, or trademarks, except as necessary to reproduce the notices described in this License.
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
 
-3.6 Termination. If you violate any term of this License, then your rights under this License (including the grants in Sections 2.1 and 2.2) will terminate immediately.
-4. Disclaimer of Warranty.
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
 
-THE WORK IS PROVIDED “AS IS” WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WARRANTIES OR CONDITIONS OF M ERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE OR NON-INFRINGEMENT. YOU BEAR THE RISK OF UNDERTAKING ANY ACTIVITIES UNDER THIS LICENSE. SOME STATES’ CONSUMER LAWS DO NOT ALLOW EXCLUSION OF AN IMPLIED WARRANTY, SO THIS DISCLAIMER MAY NOT APPLY TO YOU.
-5. Limitation of Liability.
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
 
-EXCEPT AS PROHIBITED BY APPLICABLE LAW, IN NO EVENT AND UNDER NO LEGAL THEORY, WHETHER IN TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE SHALL ANY LICENSOR BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT OF OR RELATED TO THIS LICENSE, THE USE OR INABILITY TO USE THE WORK (INCLUDING BUT NOT LIMITED TO LOSS OF GOODWILL, BUSINESS INTERRUPTION, LOST PROFITS OR DATA, COMPUTER FAILURE OR MALFUNCTION, OR ANY OTHER COMM ERCIAL DAMAGES OR LOSSES), EVEN IF THE LICENSOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
 
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

META-INF/MANIFEST.MF

@@ -2,7 +2,7 @@ Manifest-Version: 1.0
 Bundle-ManifestVersion: 2
 Bundle-Name: Amazon Kinesis Client Library for Java
 Bundle-SymbolicName: com.amazonaws.kinesisclientlibrary;singleton:=true
-Bundle-Version: 1.9.1
+Bundle-Version: 2.0.0
 Bundle-Vendor: Amazon Technologies, Inc
 Bundle-RequiredExecutionEnvironment: JavaSE-1.8
 Require-Bundle: org.apache.commons.codec;bundle-version="1.6",
@@ -15,13 +15,13 @@ Require-Bundle: org.apache.commons.codec;bundle-version="1.6",
  com.amazonaws.sdk;bundle-version="1.11.319",
 Export-Package: com.amazonaws.services.kinesis,
  com.amazonaws.services.kinesis.clientlibrary,
- com.amazonaws.services.kinesis.clientlibrary.config,
+ com.amazonaws.services.kinesis.clientlibrary.kinesisClientLibConfiguration,
  com.amazonaws.services.kinesis.clientlibrary.exceptions,
  com.amazonaws.services.kinesis.clientlibrary.exceptions.internal,
  com.amazonaws.services.kinesis.clientlibrary.interfaces,
  com.amazonaws.services.kinesis.clientlibrary.lib,
  com.amazonaws.services.kinesis.clientlibrary.lib.checkpoint,
- com.amazonaws.services.kinesis.clientlibrary.lib.worker,
+ com.amazonaws.services.kinesis.clientlibrary.lib.scheduler,
  com.amazonaws.services.kinesis.clientlibrary.proxies,
  com.amazonaws.services.kinesis.clientlibrary.types,
  com.amazonaws.services.kinesis.leases,

README.md

@@ -1,26 +1,59 @@
-# Amazon Kinesis Client Library for Java [![Build Status](https://travis-ci.org/awslabs/amazon-kinesis-client.svg?branch=master)](https://travis-ci.org/awslabs/amazon-kinesis-client)
+# Amazon Kinesis Client Library for Java
+[![Build Status](https://travis-ci.org/awslabs/amazon-kinesis-client.svg?branch=master)](https://travis-ci.org/awslabs/amazon-kinesis-client)
 
-The **Amazon Kinesis Client Library for Java** (Amazon KCL) enables Java developers to easily consume and process data from [Amazon Kinesis][kinesis].
+> [!IMPORTANT]
+> ### Amazon Kinesis Client Library (KCL) 1.x will reach end-of-support on January 30, 2026
+> Amazon Kinesis Client Library (KCL) 1.x will reach end-of-support on January 30, 2026. Accordingly, these versions will enter maintenance mode on April 17, 2025. During maintenance mode, AWS will provide updates only for critical bug fixes and security issues. Major versions in maintenance mode will not receive updates for new features or feature enhancements. If you’re using KCL 1.x, we recommend migrating to the latest versions. When migrating from KCL 1.x to 3.x, you will need to update interfaces and security credential providers in your application. For details about the end-of-support notice and required actions, see the following links:
+> * [AWS Blog: Announcing end-of-support for Amazon Kinesis Client Library 1.x and Amazon Kinesis Producer Library 0.x effective January 30, 2026](https://aws.amazon.com/blogs/big-data/announcing-end-of-support-for-amazon-kinesis-client-library-1-x-and-amazon-kinesis-producer-library-0-x-effective-january-30-2026/)
+> * [Kinesis documentation: KCL version lifecycle policy](https://docs.aws.amazon.com/streams/latest/dev/kcl-version-lifecycle-policy.html)
+> * [Kinesis documentation: Migrating from KCL 1.x to KCL 3.x](https://docs.aws.amazon.com/streams/latest/dev/kcl-migration-1-3.html)
 
-* [Kinesis Product Page][kinesis]
-* [Forum][kinesis-forum]
+## Introduction
+
+The **Amazon Kinesis Client Library (KCL) for Java** enables Java developers to easily consume and process data from [Amazon Kinesis Data Streams][kinesis].
+
+* [Kinesis Data Streams Product Page][kinesis]
+* [Amazon re:Post Forum: Kinesis][kinesis-forum]
+* [Javadoc][kcl-javadoc]
+* [FAQ](docs/FAQ.md)
+* [Developer Guide - Kinesis Client Library][kcl-aws-doc]
+* [KCL GitHub documentation](docs/) (folder)
 * [Issues][kinesis-client-library-issues]
+* [Giving Feedback][giving-feedback]
 
 ## Features
 
-* Provides an easy-to-use programming model for processing data using Amazon Kinesis
-* Helps with scale-out and fault-tolerant processing
+* **Scalability:** KCL enables applications to scale dynamically by distributing the processing load across multiple workers. You can scale your application in or out, manually or with auto-scaling, without worrying about load redistribution.
+* **Load balancing:** KCL automatically balances the processing load across available workers, resulting in an even distribution of work across workers.
+* **Checkpointing:** KCL manages checkpointing of processed records, enabling applications to resume processing from their last sucessfully processed position.
+* **Fault tolerance:** KCL provides built-in fault tolerance mechanisms, making sure that data processing continues even if individual workers fail. KCL also provides at-least-once delivery.
+* **Handling stream-level changes:** KCL adapts to shard splits and merges that might occur due to changes in data volume. It maintains ordering by making sure that child shards are processed only after their parent shard is completed and checkpointed.
+* **Monitoring:** KCL integrates with Amazon CloudWatch for consumer-level monitoring.
+* **Multi-language support:** KCL natively supports Java and enables multiple non-Java programming languages through MultiLangDaemon.
 
 ## Getting Started
 
 1. **Sign up for AWS** — Before you begin, you need an AWS account. For more information about creating an AWS account and retrieving your AWS credentials, see [AWS Account and Credentials][docs-signup] in the AWS SDK for Java Developer Guide.
-1. **Sign up for Amazon Kinesis** — Go to the Amazon Kinesis console to sign up for the service and create an Amazon Kinesis stream. For more information, see [Create an Amazon Kinesis Stream][kinesis-guide-create] in the Amazon Kinesis Developer Guide.
-1. **Minimum requirements** — To use the Amazon Kinesis Client Library, you'll need **Java 1.8+**. For more information about Amazon Kinesis Client Library requirements, see [Before You Begin][kinesis-guide-begin] in the Amazon Kinesis Developer Guide.
-1. **Using the Amazon Kinesis Client Library** — The best way to get familiar with the Amazon Kinesis Client Library is to read [Developing Record Consumer Applications][kinesis-guide-applications] in the Amazon Kinesis Developer Guide.
+2. **Sign up for Amazon Kinesis** — Go to the Amazon Kinesis console to sign up for the service and create an Amazon Kinesis stream. For more information, see [Create an Amazon Kinesis Stream][kinesis-guide-create] in the Amazon Kinesis Developer Guide.
+3. **Minimum requirements** — To use the Amazon Kinesis Client Library, you will need **Java 1.8+**. For more information about Amazon Kinesis Client Library requirements, see [Before You Begin][kinesis-guide-begin] in the Amazon Kinesis Developer Guide.
+4. **Using the Amazon Kinesis Client Library** — The best way to get familiar with the Amazon Kinesis Client Library is to read [Use Kinesis Client Library][kinesis-guide-applications] in the Amazon Kinesis Data Streams Developer Guide. For more information on core KCL concepts, please refer to the [KCL Concepts][kinesis-client-library-concepts] page.
 
 ## Building from Source
 
-After you've downloaded the code from GitHub, you can build it using Maven. To disable GPG signing in the build, use this command: `mvn clean install -Dgpg.skip=true`
+After you have downloaded the code from GitHub, you can build it using Maven. To disable GPG signing in the build, use
+this command: `mvn clean install -Dgpg.skip=true`.
+Note: This command does not run integration tests.
+
+To disable running unit tests in the build, add the property `-Dskip.ut=true`.
+
+## Running Integration Tests
+
+Note that running integration tests creates AWS resources.
+Integration tests require valid AWS credentials.
+This will look for a default AWS profile specified in your local `.aws/credentials`.
+To run all integration tests: `mvn verify -DskipITs=false`.
+To run one integration tests, specify the integration test class: `mvn -Dit.test="BasicStreamConsumerIntegrationTest" -DskipITs=false verify`
+Optionally, you can provide the name of an IAM user/role to run tests with as a string using this command: `mvn -DskipITs=false -DawsProfile="<PROFILE_NAME>" verify`.
 
 ## Integration with the Kinesis Producer Library
 For producer-side developers using the **[Kinesis Producer Library (KPL)][kinesis-guide-kpl]**, the KCL integrates without additional effort. When the KCL retrieves an aggregated Amazon Kinesis record consisting of multiple KPL user records, it will automatically invoke the KPL to extract the individual user records before returning them to the user.
@@ -28,25 +61,71 @@ For producer-side developers using the **[Kinesis Producer Library (KPL)][kinesi
 ## Amazon KCL support for other languages
 To make it easier for developers to write record processors in other languages, we have implemented a Java based daemon, called MultiLangDaemon that does all the heavy lifting. Our approach has the daemon spawn a sub-process, which in turn runs the record processor, which can be written in any language. The MultiLangDaemon process and the record processor sub-process communicate with each other over [STDIN and STDOUT using a defined protocol][multi-lang-protocol]. There will be a one to one correspondence amongst record processors, child processes, and shards. For Python developers specifically, we have abstracted these implementation details away and [expose an interface][kclpy] that enables you to focus on writing record processing logic in Python. This approach enables KCL to be language agnostic, while providing identical features and similar parallel processing model across all languages.
 
-## Release Notes
+## Using the KCL
+The recommended way to use the KCL for Java is to consume it from Maven.
 
-### Latest Release (1.9.1)
-* Added the ability to create a prepared checkpoint when at `SHARD_END`.
-  * [PR #301](https://github.com/awslabs/amazon-kinesis-client/pull/301)
-* Added the ability to subscribe to worker state change events.  
-  * [PR #291](https://github.com/awslabs/amazon-kinesis-client/pull/291)
-* Added support for custom lease managers.  
-  A custom `LeaseManager` can be provided to `Worker.Builder` that will be used to provide lease services. 
-  This makes it possible to implement custom lease management systems in addition to the default DynamoDB system.  
-  * [PR #297](https://github.com/awslabs/amazon-kinesis-client/pull/297)
-* Updated the version of the AWS Java SDK to 1.11.219
+## KCL versions
 
-### For remaining release notes check **[CHANGELOG.md][changelog-md]**.
+> [!WARNING]
+> ### Do not use AWS SDK for Java versions 2.27.19 to 2.27.23 with KCL 3.x
+> When using KCL 3.x with AWS SDK for Java versions 2.27.19 through 2.27.23, you may encounter the following DynamoDB exception:
+> ```software.amazon.awssdk.services.dynamodb.model.DynamoDbException: The document path provided in the update expression is invalid for update (Service: DynamoDb, Status Code: 400, Request ID: xxx)```.
+> This error occurs due to [a known issue](https://github.com/aws/aws-sdk-java-v2/issues/5584) in the AWS SDK for Java that affects the DynamoDB metadata table managed by KCL 3.x. The issue was introduced in version 2.27.19 and impacts all versions up to 2.27.23. The issue has been resolved in the AWS SDK for Java version 2.27.24. For optimal performance and stability, we recommend upgrading to version 2.28.0 or later.
+
+### Version 3.x
+  ``` xml
+  <dependency>
+      <groupId>software.amazon.kinesis</groupId>
+      <artifactId>amazon-kinesis-client</artifactId>
+      <version>3.0.3</version>
+  </dependency>
+  ```
+
+### Version 2.x
+[Version 2.x tracking branch](https://github.com/awslabs/amazon-kinesis-client/tree/v2.x)
+  ``` xml
+  <dependency>
+      <groupId>software.amazon.kinesis</groupId>
+      <artifactId>amazon-kinesis-client</artifactId>
+      <version>2.7.0</version>
+  </dependency>
+  ```
+### Version 1.x
+[Version 1.x tracking branch](https://github.com/awslabs/amazon-kinesis-client/tree/v1.x)
+  ``` xml
+  <dependency>
+      <groupId>com.amazonaws</groupId>
+      <artifactId>amazon-kinesis-client</artifactId>
+      <version>1.14.1</version>
+  </dependency>
+  ```
+### Release Notes
+
+| KCL Version | Changelog |
+| --- | --- |
+| 3.x | [master/CHANGELOG.md](CHANGELOG.md) |
+| 2.x | [v2.x/CHANGELOG.md](https://github.com/awslabs/amazon-kinesis-client/blob/v2.x/CHANGELOG.md) |
+| 1.x | [v1.x/CHANGELOG.md](https://github.com/awslabs/amazon-kinesis-client/blob/v1.x/CHANGELOG.md) |
+
+### Version recommendation
+We recommend all users to migrate to the latest respective versions to avoid known issues and benefit from all improvements.
+
+## Giving Feedback
+
+Help Us Improve the Kinesis Client Library! Your involvement is crucial to enhancing the Kinesis Client Library. We invite you to join our community and contribute in the following ways:
+
+* [Issue](https://github.com/awslabs/amazon-kinesis-client/issues) Reporting: This is our preferred method of communication. Use this channel to report bugs, suggest improvements, or ask questions.
+* Feature Requests: Share your ideas for new features or vote for existing proposals on our [Issues](https://github.com/awslabs/amazon-kinesis-client/issues) page. This helps us prioritize development efforts.
+* Participate in Discussions: Engage with other users and our team in our discussion forums.
+* Submit [Pull Requests](https://github.com/awslabs/amazon-kinesis-client/pulls): If you have developed a fix or improvement, we welcome your code contributions.
+
+By participating through these channels, you play a vital role in shaping the future of the Kinesis Client Library. We value your input and look forward to collaborating with you!
 
-[kinesis]: http://aws.amazon.com/kinesis
-[kinesis-forum]: http://developer.amazonwebservices.com/connect/forum.jspa?forumID=169
-[kinesis-client-library-issues]: https://github.com/awslabs/amazon-kinesis-client/issues
 [docs-signup]: http://docs.aws.amazon.com/AWSSdkDocsJava/latest/DeveloperGuide/java-dg-setup.html
+[kcl-javadoc]: https://javadoc.io/doc/software.amazon.kinesis/amazon-kinesis-client/
+[kinesis]: http://aws.amazon.com/kinesis
+[kinesis-client-library-issues]: https://github.com/awslabs/amazon-kinesis-client/issues
+[kinesis-forum]: http://developer.amazonwebservices.com/connect/forum.jspa?forumID=169
 [kinesis-guide]: http://docs.aws.amazon.com/kinesis/latest/dev/introduction.html
 [kinesis-guide-begin]: http://docs.aws.amazon.com/kinesis/latest/dev/before-you-begin.html
 [kinesis-guide-create]: http://docs.aws.amazon.com/kinesis/latest/dev/step-one-create-stream.html
@@ -55,5 +134,8 @@ To make it easier for developers to write record processors in other languages,
 [kinesis-guide-kpl]: http://docs.aws.amazon.com//kinesis/latest/dev/developing-producers-with-kpl.html
 [kinesis-guide-consumer-deaggregation]: http://docs.aws.amazon.com//kinesis/latest/dev/kinesis-kpl-consumer-deaggregation.html
 [kclpy]: https://github.com/awslabs/amazon-kinesis-client-python
-[multi-lang-protocol]: https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/multilang/package-info.java
-[changelog-md]: https://github.com/awslabs/amazon-kinesis-client/blob/master/CHANGELOG.md
+[multi-lang-protocol]: /amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/package-info.java
+[migration-guide]: https://docs.aws.amazon.com/streams/latest/dev/kcl-migration-from-previous-versions
+[kcl-sample]: https://docs.aws.amazon.com/streams/latest/dev/kcl-example-code
+[kcl-aws-doc]: https://docs.aws.amazon.com/streams/latest/dev/kcl.html
+[giving-feedback]: https://github.com/awslabs/amazon-kinesis-client?tab=readme-ov-file#giving-feedback

amazon-kinesis-client-multilang/pom.xml

@@ -0,0 +1,189 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+  -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <parent>
+    <artifactId>amazon-kinesis-client-pom</artifactId>
+    <groupId>software.amazon.kinesis</groupId>
+    <version>3.0.3</version>
+  </parent>
+  <modelVersion>4.0.0</modelVersion>
+
+  <artifactId>amazon-kinesis-client-multilang</artifactId>
+
+  <dependencies>
+    <dependency>
+      <groupId>software.amazon.kinesis</groupId>
+      <artifactId>amazon-kinesis-client</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>sts</artifactId>
+      <version>${awssdk.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.projectlombok</groupId>
+      <artifactId>lombok</artifactId>
+      <version>1.18.28</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>ch.qos.logback</groupId>
+      <artifactId>logback-classic</artifactId>
+      <version>1.3.14</version>
+    </dependency>
+    <dependency>
+      <groupId>com.beust</groupId>
+      <artifactId>jcommander</artifactId>
+      <version>1.82</version>
+    </dependency>
+    <dependency>
+      <groupId>commons-io</groupId>
+      <artifactId>commons-io</artifactId>
+      <version>2.16.1</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-collections4</artifactId>
+      <version>4.4</version>
+    </dependency>
+
+
+    <dependency>
+      <groupId>commons-beanutils</groupId>
+      <artifactId>commons-beanutils</artifactId>
+      <version>1.9.4</version>
+    </dependency>
+
+    <!-- Test -->
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-api</artifactId>
+      <version>5.11.3</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>4.13.2</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-all</artifactId>
+      <version>1.10.19</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.hamcrest</groupId>
+      <artifactId>hamcrest-all</artifactId>
+      <version>1.3</version>
+      <scope>test</scope>
+    </dependency>
+    <!-- Using older version to be compatible with Java 8   -->
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-junit-jupiter</artifactId>
+      <version>3.12.4</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <pluginManagement>
+      <plugins>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-compiler-plugin</artifactId>
+          <version>3.13.0</version>
+          <configuration>
+            <release>8</release>
+            <encoding>UTF-8</encoding>
+          </configuration>
+        </plugin>
+      </plugins>
+    </pluginManagement>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-javadoc-plugin</artifactId>
+        <version>3.7.0</version>
+        <executions>
+          <execution>
+            <id>attach-javadocs</id>
+            <goals>
+              <goal>jar</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-source-plugin</artifactId>
+        <version>3.2.1</version>
+        <executions>
+          <execution>
+            <id>attach-sources</id>
+            <goals>
+              <goal>jar</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>com.diffplug.spotless</groupId>
+        <artifactId>spotless-maven-plugin</artifactId>
+        <version>2.30.0</version> <!--last version to support java 8-->
+        <configuration>
+          <java>
+            <palantirJavaFormat />
+            <importOrder>
+              <order>java,,\#</order>
+            </importOrder>
+          </java>
+        </configuration>
+        <executions>
+          <execution>
+            <goals>
+              <goal>check</goal>
+            </goals>
+            <phase>compile</phase>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+
+  </build>
+
+  <profiles>
+    <profile>
+      <id>disable-java8-doclint</id>
+      <activation>
+        <jdk>[1.8,)</jdk>
+      </activation>
+      <properties>
+        <doclint>none</doclint>
+      </properties>
+    </profile>
+  </profiles>
+
+
+</project>

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/DrainChildSTDERRTask.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package software.amazon.kinesis.multilang;
+
+import java.io.BufferedReader;
+
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * Reads lines off the STDERR of the child process and prints them to this process's (the JVM's) STDERR and log.
+ */
+@Slf4j
+class DrainChildSTDERRTask extends LineReaderTask<Boolean> {
+    DrainChildSTDERRTask() {}
+
+    @Override
+    protected HandleLineResult<Boolean> handleLine(String line) {
+        log.error("Received error line from subprocess [{}] for shard {}", line, getShardId());
+        System.err.println(line);
+        return new HandleLineResult<Boolean>();
+    }
+
+    @Override
+    protected Boolean returnAfterException(Exception e) {
+        return false;
+    }
+
+    @Override
+    protected Boolean returnAfterEndOfInput() {
+        return true;
+    }
+
+    public LineReaderTask<Boolean> initialize(BufferedReader reader, String shardId) {
+        return initialize(reader, shardId, "Draining STDERR for " + shardId);
+    }
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/DrainChildSTDOUTTask.java

@@ -1,57 +1,53 @@
 /*
- * Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
-package com.amazonaws.services.kinesis.multilang;
+package software.amazon.kinesis.multilang;
 
 import java.io.BufferedReader;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import lombok.extern.slf4j.Slf4j;
 
 /**
  * This class is used to drain the STDOUT of the child process. After the child process has been given a shutdown
  * message and responded indicating that it is shutdown, we attempt to close the input and outputs of that process so
  * that the process can exit.
- * 
+ *
  * To understand why this is necessary, consider the following scenario:
- * 
+ *
  * <ol>
  * <li>Child process responds that it is done with shutdown.</li>
  * <li>Child process prints debugging text to STDOUT that fills the pipe buffer so child becomes blocked.</li>
  * <li>Parent process doesn't drain child process's STDOUT.</li>
  * <li>Child process remains blocked.</li>
  * </ol>
- * 
+ *
  * To prevent the child process from becoming blocked in this way, it is the responsibility of the parent process to
  * drain the child process's STDOUT. We reprint each drained line to our log to permit debugging.
  */
+@Slf4j
 class DrainChildSTDOUTTask extends LineReaderTask<Boolean> {
-
-    private static final Log LOG = LogFactory.getLog(DrainChildSTDOUTTask.class);
-
-    DrainChildSTDOUTTask() {
-    }
+    DrainChildSTDOUTTask() {}
 
     @Override
     protected HandleLineResult<Boolean> handleLine(String line) {
-        LOG.info("Drained line for shard " + getShardId() + ": " + line);
+        log.info("Drained line for shard {}: {}", getShardId(), line);
         return new HandleLineResult<Boolean>();
     }
 
     @Override
     protected Boolean returnAfterException(Exception e) {
-        LOG.info("Encountered exception while draining STDOUT of child process for shard " + getShardId(), e);
+        log.info("Encountered exception while draining STDOUT of child process for shard {}", getShardId(), e);
         return false;
     }
 

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/GetNextMessageTask.java

@@ -1,44 +1,39 @@
 /*
- * Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
-package com.amazonaws.services.kinesis.multilang;
+package software.amazon.kinesis.multilang;
 
 import java.io.BufferedReader;
 import java.io.IOException;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.amazonaws.services.kinesis.multilang.messages.Message;
-
 import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.extern.slf4j.Slf4j;
+import software.amazon.kinesis.multilang.messages.Message;
 
 /**
  * Gets the next message off the STDOUT of the child process. Throws an exception if a message is not found before the
  * end of the input stream is reached.
  */
+@Slf4j
 class GetNextMessageTask extends LineReaderTask<Message> {
-
-    private static final Log LOG = LogFactory.getLog(GetNextMessageTask.class);
-
     private ObjectMapper objectMapper;
 
     private static final String EMPTY_LINE = "";
 
     /**
      * Constructor.
-     * 
+     *
      * @param objectMapper An object mapper for decoding json messages from the input stream.
      */
     GetNextMessageTask(ObjectMapper objectMapper) {
@@ -47,7 +42,7 @@ class GetNextMessageTask extends LineReaderTask<Message> {
 
     /**
      * Checks if a line is an empty line.
-     * 
+     *
      * @param line A string
      * @return True if the line is an empty string, i.e. "", false otherwise.
      */
@@ -68,15 +63,17 @@ class GetNextMessageTask extends LineReaderTask<Message> {
                 return new HandleLineResult<Message>(objectMapper.readValue(line, Message.class));
             }
         } catch (IOException e) {
-            LOG.info("Skipping unexpected line on STDOUT for shard " + getShardId() + ": " + line);
+            log.info("Skipping unexpected line on STDOUT for shard {}: {}", getShardId(), line);
         }
         return new HandleLineResult<Message>();
     }
 
     @Override
     protected Message returnAfterException(Exception e) {
-        throw new RuntimeException("Encountered an error while reading a line from STDIN for shard " + getShardId()
-                + " so won't be able to return a message.", e);
+        throw new RuntimeException(
+                "Encountered an error while reading a line from STDIN for shard " + getShardId()
+                        + " so won't be able to return a message.",
+                e);
     }
 
     @Override

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/LineReaderTask.java

@@ -1,18 +1,18 @@
 /*
- * Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
-package com.amazonaws.services.kinesis.multilang;
+package software.amazon.kinesis.multilang;
 
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -20,8 +20,7 @@ import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.util.concurrent.Callable;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import lombok.extern.slf4j.Slf4j;
 
 /**
  * This abstract class captures the process of reading from an input stream. Three methods must be provided for
@@ -31,21 +30,18 @@ import org.apache.commons.logging.LogFactory;
  * <li> {@link #returnAfterEndOfInput()}</li>
  * <li> {@link #returnAfterException(Exception)}</li>
  * </ol>
- * 
+ *
  * @param <T>
  */
+@Slf4j
 abstract class LineReaderTask<T> implements Callable<T> {
-
-    private static final Log LOG = LogFactory.getLog(LineReaderTask.class);
-
     private BufferedReader reader;
 
     private String description;
 
     private String shardId;
 
-    LineReaderTask() {
-    }
+    LineReaderTask() {}
 
     /**
      * Reads lines off the input stream until a return value is set, or an exception is encountered, or the end of the
@@ -56,7 +52,7 @@ abstract class LineReaderTask<T> implements Callable<T> {
     public T call() throws Exception {
         String nextLine = null;
         try {
-            LOG.info("Starting: " + description);
+            log.info("Starting: {}", description);
             while ((nextLine = reader.readLine()) != null) {
                 HandleLineResult<T> result = handleLine(nextLine);
                 if (result.hasReturnValue()) {
@@ -66,7 +62,7 @@ abstract class LineReaderTask<T> implements Callable<T> {
         } catch (IOException e) {
             return returnAfterException(e);
         }
-        LOG.info("Stopping: " + description);
+        log.info("Stopping: {}", description);
         return returnAfterEndOfInput();
     }
 
@@ -75,7 +71,7 @@ abstract class LineReaderTask<T> implements Callable<T> {
      * return from the {@link #call()} function by having a value, indicating that value should be returned immediately
      * without reading further, or not having a value, indicating that more lines of input need to be read before
      * returning.
-     * 
+     *
      * @param line A line read from the input stream.
      * @return HandleLineResult<T> which may or may not have a has return value, indicating to return or not return yet
      *         respectively.
@@ -86,7 +82,7 @@ abstract class LineReaderTask<T> implements Callable<T> {
      * This method will be called if there is an error while reading from the input stream. The return value of this
      * method will be returned as the result of this Callable unless an Exception is thrown. If an Exception is thrown
      * then that exception will be thrown by the Callable.
-     * 
+     *
      * @param e An exception that occurred while reading from the input stream.
      * @return What to return.
      */
@@ -96,7 +92,7 @@ abstract class LineReaderTask<T> implements Callable<T> {
      * This method will be called once the end of the input stream is reached. The return value of this method will be
      * returned as the result of this Callable. Implementations of this method are welcome to throw a runtime exception
      * to indicate that the task was unsuccessful.
-     * 
+     *
      * @return What to return.
      */
     protected abstract T returnAfterEndOfInput();
@@ -104,7 +100,7 @@ abstract class LineReaderTask<T> implements Callable<T> {
     /**
      * Allows subclasses to provide more detailed logs. Specifically, this allows the drain tasks and GetNextMessageTask
      * to log which shard they're working on.
-     * 
+     *
      * @return The shard id
      */
     public String getShardId() {
@@ -113,7 +109,7 @@ abstract class LineReaderTask<T> implements Callable<T> {
 
     /**
      * The description should be a string explaining what this particular LineReader class does.
-     * 
+     *
      * @return The description.
      */
     public String getDescription() {
@@ -124,7 +120,7 @@ abstract class LineReaderTask<T> implements Callable<T> {
      * The result of a call to {@link LineReaderTask#handleLine(String)}. Allows implementations of that method to
      * indicate whether a particular invocation of that method produced a return for this task or not. If a return value
      * doesn't exist the {@link #call()} method will continue to the next line.
-     * 
+     *
      * @param <V>
      */
     protected class HandleLineResult<V> {
@@ -157,11 +153,11 @@ abstract class LineReaderTask<T> implements Callable<T> {
     /**
      * An initialization method allows us to delay setting the attributes of this class. Some of the attributes, stream
      * and shardId, are not known to the {@link MultiLangRecordProcessorFactory} when it constructs a
-     * {@link MultiLangRecordProcessor} but are later determined when
-     * {@link MultiLangRecordProcessor#initialize(String)} is called. So we follow a pattern where the attributes are
+     * {@link MultiLangShardRecordProcessor} but are later determined when
+     * {@link MultiLangShardRecordProcessor#initialize(String)} is called. So we follow a pattern where the attributes are
      * set inside this method instead of the constructor so that this object will be initialized when all its attributes
      * are known to the record processor.
-     * 
+     *
      * @param stream
      * @param shardId
      * @param description
@@ -183,5 +179,4 @@ abstract class LineReaderTask<T> implements Callable<T> {
         this.description = description;
         return this;
     }
-
 }

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MessageReader.java

@@ -1,18 +1,18 @@
 /*
- * Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
-package com.amazonaws.services.kinesis.multilang;
+package software.amazon.kinesis.multilang;
 
 import java.io.BufferedReader;
 import java.io.InputStream;
@@ -20,19 +20,19 @@ import java.io.InputStreamReader;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
 
-import com.amazonaws.services.kinesis.multilang.messages.Message;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import software.amazon.kinesis.multilang.messages.Message;
 
 /**
  * Provides methods for interacting with the child process's STDOUT.
- * 
+ *
  * {@link #getNextMessageFromSTDOUT()} reads lines from the child process's STDOUT and attempts to decode a
  * {@link Message} object from each line. A child process's STDOUT could have lines that don't contain data related to
  * the multi-language protocol, such as when the child process prints debugging information to its STDOUT (instead of
  * logging to a file), also when a child processes writes a Message it is expected to prepend and append a new line
  * character to their message to help ensure that it is isolated on a line all by itself which results in empty lines
  * being present in STDOUT. Lines which cannot be decoded to a Message object are ignored.
- * 
+ *
  * {@link #drainSTDOUT()} simply reads all data from the child process's STDOUT until the stream is closed.
  */
 class MessageReader {
@@ -48,19 +48,18 @@ class MessageReader {
     /**
      * Use the initialize methods after construction.
      */
-    MessageReader() {
-    }
+    MessageReader() {}
 
     /**
      * Returns a future which represents an attempt to read the next message in the child process's STDOUT. If the task
      * is successful, the result of the future will be the next message found in the child process's STDOUT, if the task
      * is unable to find a message before the child process's STDOUT is closed, or reading from STDOUT causes an
      * IOException, then an execution exception will be generated by this future.
-     * 
+     *
      * The task employed by this method reads from the child process's STDOUT line by line. The task attempts to decode
      * each line into a {@link Message} object. Lines that fail to decode to a Message are ignored and the task
      * continues to the next line until it finds a Message.
-     * 
+     *
      * @return
      */
     Future<Message> getNextMessageFromSTDOUT() {
@@ -73,7 +72,7 @@ class MessageReader {
      * Returns a future that represents a computation that drains the STDOUT of the child process. That future's result
      * is true if the end of the child's STDOUT is reached, its result is false if there was an error while reading from
      * the stream. This task will log all the lines it drains to permit debugging.
-     * 
+     *
      * @return
      */
     Future<Boolean> drainSTDOUT() {
@@ -85,23 +84,20 @@ class MessageReader {
     /**
      * An initialization method allows us to delay setting the attributes of this class. Some of the attributes,
      * stream and shardId, are not known to the {@link MultiLangRecordProcessorFactory} when it constructs a
-     * {@link MultiLangRecordProcessor} but are later determined when
-     * {@link MultiLangRecordProcessor#initialize(String)} is called. So we follow a pattern where the attributes are
+     * {@link MultiLangShardRecordProcessor} but are later determined when
+     * {@link MultiLangShardRecordProcessor#initialize(String)} is called. So we follow a pattern where the attributes are
      * set inside this method instead of the constructor so that this object will be initialized when all its attributes
      * are known to the record processor.
-     * 
+     *
      * @param stream Used to read messages from the subprocess.
      * @param shardId The shard we're working on.
      * @param objectMapper The object mapper to decode messages.
      * @param executorService An executor service to run tasks in.
      */
-    MessageReader initialize(InputStream stream,
-            String shardId,
-            ObjectMapper objectMapper,
-            ExecutorService executorService) {
-        return this.initialize(new BufferedReader(new InputStreamReader(stream)), shardId, objectMapper,
-                executorService);
-
+    MessageReader initialize(
+            InputStream stream, String shardId, ObjectMapper objectMapper, ExecutorService executorService) {
+        return this.initialize(
+                new BufferedReader(new InputStreamReader(stream)), shardId, objectMapper, executorService);
     }
 
     /**
@@ -110,10 +106,8 @@ class MessageReader {
      * @param objectMapper The object mapper to decode messages.
      * @param executorService An executor service to run tasks in.
      */
-    MessageReader initialize(BufferedReader reader,
-            String shardId,
-            ObjectMapper objectMapper,
-            ExecutorService executorService) {
+    MessageReader initialize(
+            BufferedReader reader, String shardId, ObjectMapper objectMapper, ExecutorService executorService) {
         this.reader = reader;
         this.shardId = shardId;
         this.objectMapper = objectMapper;

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MessageWriter.java

@@ -1,18 +1,18 @@
 /*
- * Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
-package com.amazonaws.services.kinesis.multilang;
+package software.amazon.kinesis.multilang;
 
 import java.io.BufferedWriter;
 import java.io.IOException;
@@ -22,29 +22,25 @@ import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.amazonaws.services.kinesis.clientlibrary.lib.worker.ShutdownReason;
-
-import com.amazonaws.services.kinesis.multilang.messages.CheckpointMessage;
-import com.amazonaws.services.kinesis.multilang.messages.InitializeMessage;
-import com.amazonaws.services.kinesis.multilang.messages.Message;
-import com.amazonaws.services.kinesis.multilang.messages.ProcessRecordsMessage;
-import com.amazonaws.services.kinesis.multilang.messages.ShutdownMessage;
-import com.amazonaws.services.kinesis.multilang.messages.ShutdownRequestedMessage;
-
-import com.amazonaws.services.kinesis.clientlibrary.types.InitializationInput;
-import com.amazonaws.services.kinesis.clientlibrary.types.ProcessRecordsInput;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.extern.slf4j.Slf4j;
+import software.amazon.kinesis.lifecycle.events.InitializationInput;
+import software.amazon.kinesis.lifecycle.events.LeaseLostInput;
+import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput;
+import software.amazon.kinesis.lifecycle.events.ShardEndedInput;
+import software.amazon.kinesis.multilang.messages.CheckpointMessage;
+import software.amazon.kinesis.multilang.messages.InitializeMessage;
+import software.amazon.kinesis.multilang.messages.LeaseLostMessage;
+import software.amazon.kinesis.multilang.messages.Message;
+import software.amazon.kinesis.multilang.messages.ProcessRecordsMessage;
+import software.amazon.kinesis.multilang.messages.ShardEndedMessage;
+import software.amazon.kinesis.multilang.messages.ShutdownRequestedMessage;
 
 /**
  * Defines methods for writing {@link Message} objects to the child process's STDIN.
  */
+@Slf4j
 class MessageWriter {
-
-    private static final Log LOG = LogFactory.getLog(MessageWriter.class);
-
     private BufferedWriter writer;
 
     private volatile boolean open = true;
@@ -58,13 +54,12 @@ class MessageWriter {
     /**
      * Use initialize method after construction.
      */
-    MessageWriter() {
-    }
+    MessageWriter() {}
 
     /**
      * Writes the message then writes the line separator provided by the system. Flushes each message to guarantee it
      * is delivered as soon as possible to the subprocess.
-     * 
+     *
      * @param message A message to be written to the subprocess.
      * @return
      * @throws IOException
@@ -79,10 +74,13 @@ class MessageWriter {
                      */
                     synchronized (writer) {
                         writer.write(message, 0, message.length());
-                        writer.write(System.lineSeparator(), 0, System.lineSeparator().length());
+                        writer.write(
+                                System.lineSeparator(),
+                                0,
+                                System.lineSeparator().length());
                         writer.flush();
                     }
-                    LOG.info("Message size == " + message.getBytes().length + " bytes for shard " + shardId);
+                    log.info("Message size == {} bytes for shard {}", message.getBytes().length, shardId);
                 } catch (IOException e) {
                     open = false;
                 }
@@ -94,34 +92,34 @@ class MessageWriter {
             return this.executorService.submit(writeMessageToOutputTask);
         } else {
             String errorMessage = "Cannot write message " + message + " because writer is closed for shard " + shardId;
-            LOG.info(errorMessage);
+            log.info(errorMessage);
             throw new IllegalStateException(errorMessage);
         }
     }
 
     /**
      * Converts the message to a JSON string and writes it to the subprocess.
-     * 
+     *
      * @param message A message to be written to the subprocess.
      * @return
      */
     private Future<Boolean> writeMessage(Message message) {
-        LOG.info("Writing " + message.getClass().getSimpleName() + " to child process for shard " + shardId);
+        log.info("Writing {} to child process for shard {}", message.getClass().getSimpleName(), shardId);
         try {
             String jsonText = objectMapper.writeValueAsString(message);
             return writeMessageToOutput(jsonText);
         } catch (IOException e) {
-            String errorMessage =
-                    String.format("Encountered I/O error while writing %s action to subprocess", message.getClass()
-                            .getSimpleName());
-            LOG.error(errorMessage, e);
+            String errorMessage = String.format(
+                    "Encountered I/O error while writing %s action to subprocess",
+                    message.getClass().getSimpleName());
+            log.error(errorMessage, e);
             throw new RuntimeException(errorMessage, e);
         }
     }
 
     /**
      * Writes an {@link InitializeMessage} to the subprocess.
-     * 
+     *
      * @param initializationInput
      *            contains information about the shard being initialized
      */
@@ -131,7 +129,7 @@ class MessageWriter {
 
     /**
      * Writes a {@link ProcessRecordsMessage} message to the subprocess.
-     * 
+     *
      * @param processRecordsInput
      *            the records, and associated metadata to be processed.
      */
@@ -140,12 +138,25 @@ class MessageWriter {
     }
 
     /**
-     * Writes a {@link ShutdownMessage} to the subprocess.
-     * 
-     * @param reason The reason for shutting down.
+     * Writes the lease lost message to the sub process.
+     *
+     * @param leaseLostInput
+     *            the lease lost input. This is currently unused as lease loss doesn't actually have anything in it
+     * @return A future that is set when the message has been written.
      */
-    Future<Boolean> writeShutdownMessage(ShutdownReason reason) {
-        return writeMessage(new ShutdownMessage(reason));
+    Future<Boolean> writeLeaseLossMessage(@SuppressWarnings("unused") LeaseLostInput leaseLostInput) {
+        return writeMessage(new LeaseLostMessage());
+    }
+
+    /**
+     * Writes a message to the sub process indicating that the shard has ended
+     *
+     * @param shardEndedInput
+     *            the shard end input. This is currently unused as the checkpoint is extracted, and used by the caller.
+     * @return A future that is set when the message has been written.
+     */
+    Future<Boolean> writeShardEndedMessage(@SuppressWarnings("unused") ShardEndedInput shardEndedInput) {
+        return writeMessage(new ShardEndedMessage());
     }
 
     /**
@@ -157,7 +168,7 @@ class MessageWriter {
 
     /**
      * Writes a {@link CheckpointMessage} to the subprocess.
-     * 
+     *
      * @param sequenceNumber
      *            The sequence number that was checkpointed.
      * @param subSequenceNumber
@@ -165,14 +176,14 @@ class MessageWriter {
      * @param throwable
      *            The exception that was thrown by a checkpoint attempt. Null if one didn't occur.
      */
-    Future<Boolean> writeCheckpointMessageWithError(String sequenceNumber, Long subSequenceNumber,
-            Throwable throwable) {
+    Future<Boolean> writeCheckpointMessageWithError(
+            String sequenceNumber, Long subSequenceNumber, Throwable throwable) {
         return writeMessage(new CheckpointMessage(sequenceNumber, subSequenceNumber, throwable));
     }
 
     /**
      * Closes the output stream and prevents further attempts to write.
-     * 
+     *
      * @throws IOException Thrown when closing the writer fails
      */
     void close() throws IOException {
@@ -187,22 +198,20 @@ class MessageWriter {
     /**
      * An initialization method allows us to delay setting the attributes of this class. Some of the attributes,
      * stream and shardId, are not known to the {@link MultiLangRecordProcessorFactory} when it constructs a
-     * {@link MultiLangRecordProcessor} but are later determined when
-     * {@link MultiLangRecordProcessor#initialize(String)} is called. So we follow a pattern where the attributes are
+     * {@link MultiLangShardRecordProcessor} but are later determined when
+     * {@link MultiLangShardRecordProcessor (String)} is called. So we follow a pattern where the attributes are
      * set inside this method instead of the constructor so that this object will be initialized when all its attributes
      * are known to the record processor.
-     * 
+     *
      * @param stream Used to write messages to the subprocess.
      * @param shardId The shard we're working on.
      * @param objectMapper The object mapper to encode messages.
      * @param executorService An executor service to run tasks in.
      */
-    MessageWriter initialize(OutputStream stream,
-            String shardId,
-            ObjectMapper objectMapper,
-            ExecutorService executorService) {
-        return this.initialize(new BufferedWriter(new OutputStreamWriter(stream)), shardId, objectMapper,
-                executorService);
+    MessageWriter initialize(
+            OutputStream stream, String shardId, ObjectMapper objectMapper, ExecutorService executorService) {
+        return this.initialize(
+                new BufferedWriter(new OutputStreamWriter(stream)), shardId, objectMapper, executorService);
     }
 
     /**
@@ -211,15 +220,12 @@ class MessageWriter {
      * @param objectMapper The object mapper to encode messages.
      * @param executorService An executor service to run tasks in.
      */
-    MessageWriter initialize(BufferedWriter writer,
-            String shardId,
-            ObjectMapper objectMapper,
-            ExecutorService executorService) {
+    MessageWriter initialize(
+            BufferedWriter writer, String shardId, ObjectMapper objectMapper, ExecutorService executorService) {
         this.writer = writer;
         this.shardId = shardId;
         this.objectMapper = objectMapper;
         this.executorService = executorService;
         return this;
     }
-
 }

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MultiLangDaemon.java

@@ -0,0 +1,235 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package software.amazon.kinesis.multilang;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import ch.qos.logback.classic.LoggerContext;
+import ch.qos.logback.classic.joran.JoranConfigurator;
+import ch.qos.logback.core.joran.spi.JoranException;
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+import lombok.Data;
+import lombok.experimental.Accessors;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.LoggerFactory;
+import software.amazon.kinesis.coordinator.Scheduler;
+
+/**
+ * Main app that launches the scheduler that runs the multi-language record processor.
+ *
+ * Requires a properties file containing configuration for this daemon and the KCL. A properties file should at minimum
+ * define these properties:
+ *
+ * <pre>
+ * # The script that abides by the multi-language protocol. This script will
+ * # be executed by the MultiLangDaemon, which will communicate with this script
+ * # over STDIN and STDOUT according to the multi-language protocol.
+ * executableName = sampleapp.py
+ *
+ * # The name of an Amazon Kinesis stream to process.
+ * streamName = words
+ *
+ * # Used by the KCL as the name of this application. Will be used as the name
+ * # of a Amazon DynamoDB table which will store the lease and checkpoint
+ * # information for workers with this application name.
+ * applicationName = PythonKCLSample
+ *
+ * # Users can change the credentials provider the KCL will use to retrieve credentials.
+ * # The DefaultCredentialsProvider checks several other providers, which is
+ * # described here:
+ * # https://sdk.amazonaws.com/java/api/2.0.0-preview-11/software/amazon/awssdk/auth/credentials/DefaultCredentialsProvider.html
+ * AwsCredentialsProvider = DefaultCredentialsProvider
+ * </pre>
+ */
+@Slf4j
+public class MultiLangDaemon {
+    static class MultiLangDaemonArguments {
+        @Parameter
+        List<String> parameters = new ArrayList<>();
+
+        @Parameter(
+                names = {"-p", "--properties-file"},
+                description = "Properties file to be used with the KCL")
+        String propertiesFile;
+
+        @Parameter(
+                names = {"-l", "--log-configuration"},
+                description = "File location of logback.xml to be override the default")
+        String logConfiguration;
+    }
+
+    @Data
+    @Accessors(fluent = true)
+    static class MultiLangRunner implements Callable<Integer> {
+        private final Scheduler scheduler;
+
+        @Override
+        public Integer call() throws Exception {
+            int exitCode = 0;
+            try {
+                scheduler().run();
+            } catch (Throwable t) {
+                log.error("Caught throwable while processing data", t);
+                exitCode = 1;
+            }
+            return exitCode;
+        }
+    }
+
+    JCommander buildJCommanderAndParseArgs(final MultiLangDaemonArguments arguments, final String[] args) {
+        JCommander jCommander = JCommander.newBuilder()
+                .programName("amazon-kinesis-client MultiLangDaemon")
+                .addObject(arguments)
+                .build();
+        jCommander.parse(args);
+        return jCommander;
+    }
+
+    void printUsage(final JCommander jCommander, final String message) {
+        if (StringUtils.isNotEmpty(message)) {
+            System.err.println(message);
+        }
+        jCommander.usage();
+    }
+
+    Scheduler buildScheduler(final MultiLangDaemonConfig config) {
+        return config.getMultiLangDaemonConfiguration().build(config.getRecordProcessorFactory());
+    }
+
+    void configureLogging(final String logConfiguration) {
+        if (StringUtils.isNotEmpty(logConfiguration)) {
+            LoggerContext loggerContext = (LoggerContext) LoggerFactory.getILoggerFactory();
+            JoranConfigurator configurator = new JoranConfigurator();
+            configureLogging(logConfiguration, loggerContext, configurator);
+        }
+    }
+
+    void configureLogging(
+            final String logConfiguration, final LoggerContext loggerContext, final JoranConfigurator configurator) {
+        loggerContext.reset();
+        try (InputStream inputStream = FileUtils.openInputStream(new File(logConfiguration))) {
+            configurator.setContext(loggerContext);
+            configurator.doConfigure(inputStream);
+        } catch (IOException | JoranException e) {
+            throw new RuntimeException("Error while loading log configuration: " + e.getMessage());
+        }
+    }
+
+    String validateAndGetPropertiesFileName(final MultiLangDaemonArguments arguments) {
+        String propertiesFile = "";
+
+        if (CollectionUtils.isNotEmpty(arguments.parameters)) {
+            if (arguments.parameters.size() == 1) {
+                propertiesFile = arguments.parameters.get(0);
+            } else {
+                throw new RuntimeException("Expected a single argument, but found multiple arguments.  Arguments: "
+                        + String.join(", ", arguments.parameters));
+            }
+        }
+
+        if (StringUtils.isNotEmpty(arguments.propertiesFile)) {
+            if (StringUtils.isNotEmpty(propertiesFile)) {
+                log.warn("Overriding the properties file with the --properties-file option");
+            }
+            propertiesFile = arguments.propertiesFile;
+        }
+
+        if (StringUtils.isEmpty(propertiesFile)) {
+            throw new RuntimeException("Properties file missing, please provide a properties file");
+        }
+
+        return propertiesFile;
+    }
+
+    MultiLangDaemonConfig buildMultiLangDaemonConfig(final String propertiesFile) {
+        try {
+            return new MultiLangDaemonConfig(propertiesFile);
+        } catch (IOException e) {
+            throw new RuntimeException("Error while reading properties file: " + e.getMessage());
+        }
+    }
+
+    void setupShutdownHook(final Runtime runtime, final MultiLangRunner runner, final MultiLangDaemonConfig config) {
+        long shutdownGraceMillis = config.getMultiLangDaemonConfiguration().getShutdownGraceMillis();
+        runtime.addShutdownHook(new Thread(() -> {
+            log.info("Process terminated, will initiate shutdown.");
+            try {
+                Future<Boolean> runnerFuture = runner.scheduler().startGracefulShutdown();
+                runnerFuture.get(shutdownGraceMillis, TimeUnit.MILLISECONDS);
+                log.info("Process shutdown is complete.");
+            } catch (InterruptedException | ExecutionException | TimeoutException e) {
+                log.error("Encountered an error during shutdown.", e);
+            }
+        }));
+    }
+
+    int submitRunnerAndWait(final MultiLangDaemonConfig config, final MultiLangRunner runner) {
+        ExecutorService executorService = config.getExecutorService();
+        Future<Integer> future = executorService.submit(runner);
+
+        try {
+            return future.get();
+        } catch (InterruptedException | ExecutionException e) {
+            log.error("Encountered an error while running daemon", e);
+        }
+        return 1;
+    }
+
+    void exit(final int exitCode) {
+        System.exit(exitCode);
+    }
+
+    /**
+     * @param args
+     *            Accepts a single argument, that argument is a properties file which provides KCL configuration as
+     *            well as the name of an executable.
+     */
+    public static void main(final String[] args) {
+        int exitCode = 1;
+        MultiLangDaemon daemon = new MultiLangDaemon();
+        MultiLangDaemonArguments arguments = new MultiLangDaemonArguments();
+        JCommander jCommander = daemon.buildJCommanderAndParseArgs(arguments, args);
+        try {
+            String propertiesFileName = daemon.validateAndGetPropertiesFileName(arguments);
+            daemon.configureLogging(arguments.logConfiguration);
+            MultiLangDaemonConfig config = daemon.buildMultiLangDaemonConfig(propertiesFileName);
+
+            Scheduler scheduler = daemon.buildScheduler(config);
+            MultiLangRunner runner = new MultiLangRunner(scheduler);
+
+            daemon.setupShutdownHook(Runtime.getRuntime(), runner, config);
+            exitCode = daemon.submitRunnerAndWait(config, runner);
+        } catch (Throwable t) {
+            t.printStackTrace(System.err);
+            daemon.printUsage(jCommander, t.getMessage());
+            System.err.println("For more information, visit: https://github.com/awslabs/amazon-kinesis-client");
+        }
+        daemon.exit(exitCode);
+    }
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MultiLangDaemonConfig.java

@@ -1,18 +1,18 @@
 /*
- *  Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- *  Licensed under the Amazon Software License (the "License").
- *  You may not use this file except in compliance with the License.
- *  A copy of the License is located at
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
- *  http://aws.amazon.com/asl/
- *
- *  or in the "license" file accompanying this file. This file is distributed
- *  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- *  express or implied. See the License for the specific language governing
- *  permissions and limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
-package com.amazonaws.services.kinesis.multilang;
+package software.amazon.kinesis.multilang;
 
 import java.io.File;
 import java.io.FileInputStream;
@@ -26,20 +26,17 @@ import java.util.concurrent.SynchronousQueue;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.amazonaws.services.kinesis.clientlibrary.config.KinesisClientLibConfigurator;
-import com.amazonaws.services.kinesis.clientlibrary.lib.worker.KinesisClientLibConfiguration;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import lombok.extern.slf4j.Slf4j;
+import software.amazon.kinesis.multilang.config.KinesisClientLibConfigurator;
+import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
+import software.amazon.kinesis.retrieval.RetrievalConfig;
 
 /**
  * This class captures the configuration needed to run the MultiLangDaemon.
  */
+@Slf4j
 public class MultiLangDaemonConfig {
-
-    private static final Log LOG = LogFactory.getLog(MultiLangDaemonConfig.class);
-
     private static final String USER_AGENT = "amazon-kinesis-multi-lang-daemon";
     private static final String VERSION = "1.0.1";
 
@@ -47,63 +44,77 @@ public class MultiLangDaemonConfig {
     private static final String PROP_PROCESSING_LANGUAGE = "processingLanguage";
     private static final String PROP_MAX_ACTIVE_THREADS = "maxActiveThreads";
 
-    private KinesisClientLibConfiguration kinesisClientLibConfig;
+    private final MultiLangDaemonConfiguration multiLangDaemonConfiguration;
 
-    private ExecutorService executorService;
+    private final ExecutorService executorService;
 
-    private MultiLangRecordProcessorFactory recordProcessorFactory;
+    private final MultiLangRecordProcessorFactory recordProcessorFactory;
 
     /**
      * Constructor.
-     * 
-     * @param propertiesFile The location of the properties file.
-     * @throws IOException Thrown when the properties file can't be accessed.
-     * @throws IllegalArgumentException Thrown when the contents of the properties file are not as expected.
+     *
+     * @param propertiesFile
+     *            The location of the properties file.
+     * @throws IOException
+     *             Thrown when the properties file can't be accessed.
+     * @throws IllegalArgumentException
+     *             Thrown when the contents of the properties file are not as expected.
      */
     public MultiLangDaemonConfig(String propertiesFile) throws IOException, IllegalArgumentException {
         this(propertiesFile, Thread.currentThread().getContextClassLoader());
     }
 
     /**
-     * 
-     * @param propertiesFile The location of the properties file.
-     * @param classLoader A classloader, useful if trying to programmatically configure with the daemon, such as in a
-     *        unit test.
-     * @throws IOException Thrown when the properties file can't be accessed.
-     * @throws IllegalArgumentException Thrown when the contents of the properties file are not as expected.
+     *
+     * @param propertiesFile
+     *            The location of the properties file.
+     * @param classLoader
+     *            A classloader, useful if trying to programmatically configure with the daemon, such as in a unit test.
+     * @throws IOException
+     *             Thrown when the properties file can't be accessed.
+     * @throws IllegalArgumentException
+     *             Thrown when the contents of the properties file are not as expected.
      */
-    public MultiLangDaemonConfig(String propertiesFile, ClassLoader classLoader) throws IOException,
-            IllegalArgumentException {
+    public MultiLangDaemonConfig(String propertiesFile, ClassLoader classLoader)
+            throws IOException, IllegalArgumentException {
         this(propertiesFile, classLoader, new KinesisClientLibConfigurator());
     }
 
     /**
-     * 
-     * @param propertiesFile The location of the properties file.
-     * @param classLoader A classloader, useful if trying to programmatically configure with the daemon, such as in a
-     *        unit test.
-     * @param configurator A configurator to use.
-     * @throws IOException Thrown when the properties file can't be accessed.
-     * @throws IllegalArgumentException Thrown when the contents of the properties file are not as expected.
+     *
+     * @param propertiesFile
+     *            The location of the properties file.
+     * @param classLoader
+     *            A classloader, useful if trying to programmatically configure with the daemon, such as in a unit test.
+     * @param configurator
+     *            A configurator to use.
+     * @throws IOException
+     *             Thrown when the properties file can't be accessed.
+     * @throws IllegalArgumentException
+     *             Thrown when the contents of the properties file are not as expected.
      */
-    public MultiLangDaemonConfig(String propertiesFile,
-            ClassLoader classLoader,
-            KinesisClientLibConfigurator configurator) throws IOException, IllegalArgumentException {
+    public MultiLangDaemonConfig(
+            String propertiesFile, ClassLoader classLoader, KinesisClientLibConfigurator configurator)
+            throws IOException, IllegalArgumentException {
         Properties properties = loadProperties(classLoader, propertiesFile);
         if (!validateProperties(properties)) {
-            throw new IllegalArgumentException("Must provide an executable name in the properties file, "
-                    + "e.g. executableName = sampleapp.py");
+            throw new IllegalArgumentException(
+                    "Must provide an executable name in the properties file, " + "e.g. executableName = sampleapp.py");
         }
 
         String executableName = properties.getProperty(PROP_EXECUTABLE_NAME);
         String processingLanguage = properties.getProperty(PROP_PROCESSING_LANGUAGE);
 
-        kinesisClientLibConfig = configurator.getConfiguration(properties);
+        multiLangDaemonConfiguration = configurator.getConfiguration(properties);
         executorService = buildExecutorService(properties);
-        recordProcessorFactory = new MultiLangRecordProcessorFactory(executableName, executorService, kinesisClientLibConfig);
+        recordProcessorFactory =
+                new MultiLangRecordProcessorFactory(executableName, executorService, multiLangDaemonConfiguration);
 
-        LOG.info("Running " + kinesisClientLibConfig.getApplicationName() + " to process stream "
-                + kinesisClientLibConfig.getStreamName() + " with executable " + executableName);
+        log.info(
+                "Running {} to process stream {} with executable {}",
+                multiLangDaemonConfiguration.getApplicationName(),
+                multiLangDaemonConfiguration.getStreamName(),
+                executableName);
         prepare(processingLanguage);
     }
 
@@ -111,11 +122,9 @@ public class MultiLangDaemonConfig {
         // Ensure the JVM will refresh the cached IP values of AWS resources (e.g. service endpoints).
         java.security.Security.setProperty("networkaddress.cache.ttl", "60");
 
-        LOG.info("Using workerId: " + kinesisClientLibConfig.getWorkerIdentifier());
-        LOG.info("Using credentials with access key id: "
-                + kinesisClientLibConfig.getKinesisCredentialsProvider().getCredentials().getAWSAccessKeyId());
+        log.info("Using workerId: {}", multiLangDaemonConfiguration.getWorkerIdentifier());
 
-        StringBuilder userAgent = new StringBuilder(KinesisClientLibConfiguration.KINESIS_CLIENT_LIB_USER_AGENT);
+        StringBuilder userAgent = new StringBuilder(RetrievalConfig.KINESIS_CLIENT_LIB_USER_AGENT);
         userAgent.append(" ");
         userAgent.append(USER_AGENT);
         userAgent.append("/");
@@ -131,9 +140,8 @@ public class MultiLangDaemonConfig {
             userAgent.append(recordProcessorFactory.getCommandArray()[0]);
         }
 
-        LOG.info(String.format("MultiLangDaemon is adding the following fields to the User Agent: %s",
-                userAgent.toString()));
-        kinesisClientLibConfig.withUserAgent(userAgent.toString());
+        log.info("MultiLangDaemon is adding the following fields to the User Agent: {}", userAgent.toString());
+        //        multiLangDaemonConfiguration.withUserAgent(userAgent.toString());
     }
 
     private static Properties loadProperties(ClassLoader classLoader, String propertiesFileName) throws IOException {
@@ -160,7 +168,6 @@ public class MultiLangDaemonConfig {
                 propertyStream.close();
             }
         }
-
     }
 
     private static boolean validateProperties(Properties properties) {
@@ -174,28 +181,33 @@ public class MultiLangDaemonConfig {
     private static ExecutorService buildExecutorService(Properties properties) {
         int maxActiveThreads = getMaxActiveThreads(properties);
         ThreadFactoryBuilder builder = new ThreadFactoryBuilder().setNameFormat("multi-lang-daemon-%04d");
-        LOG.debug(String.format("Value for %s property is %d", PROP_MAX_ACTIVE_THREADS, maxActiveThreads));
+        log.debug("Value for {} property is {}", PROP_MAX_ACTIVE_THREADS, maxActiveThreads);
         if (maxActiveThreads <= 0) {
-            LOG.info("Using a cached thread pool.");
-            return new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
-                    builder.build());
+            log.info("Using a cached thread pool.");
+            return new ThreadPoolExecutor(
+                    0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, new SynchronousQueue<>(), builder.build());
         } else {
-            LOG.info(String.format("Using a fixed thread pool with %d max active threads.", maxActiveThreads));
-            return new ThreadPoolExecutor(maxActiveThreads, maxActiveThreads, 0L, TimeUnit.MILLISECONDS,
-                    new LinkedBlockingQueue<Runnable>(), builder.build());
+            log.info("Using a fixed thread pool with {} max active threads.", maxActiveThreads);
+            return new ThreadPoolExecutor(
+                    maxActiveThreads,
+                    maxActiveThreads,
+                    0L,
+                    TimeUnit.MILLISECONDS,
+                    new LinkedBlockingQueue<>(),
+                    builder.build());
         }
     }
 
     /**
-     * 
+     *
      * @return A KinesisClientLibConfiguration object based on the properties file provided.
      */
-    public KinesisClientLibConfiguration getKinesisClientLibConfiguration() {
-        return kinesisClientLibConfig;
+    public MultiLangDaemonConfiguration getMultiLangDaemonConfiguration() {
+        return multiLangDaemonConfiguration;
     }
 
     /**
-     * 
+     *
      * @return An executor service based on the properties file provided.
      */
     public ExecutorService getExecutorService() {
@@ -203,7 +215,7 @@ public class MultiLangDaemonConfig {
     }
 
     /**
-     * 
+     *
      * @return A MultiLangRecordProcessorFactory based on the properties file provided.
      */
     public MultiLangRecordProcessorFactory getRecordProcessorFactory() {

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MultiLangProtocol.java

@@ -1,33 +1,18 @@
 /*
- *  Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- *  Licensed under the Amazon Software License (the "License").
- *  You may not use this file except in compliance with the License.
- *  A copy of the License is located at
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
- *  http://aws.amazon.com/asl/
- *
- *  or in the "license" file accompanying this file. This file is distributed
- *  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- *  express or implied. See the License for the specific language governing
- *  permissions and limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
-package com.amazonaws.services.kinesis.multilang;
-
-import com.amazonaws.services.kinesis.clientlibrary.exceptions.InvalidStateException;
-import com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorCheckpointer;
-import com.amazonaws.services.kinesis.clientlibrary.lib.worker.KinesisClientLibConfiguration;
-import com.amazonaws.services.kinesis.clientlibrary.lib.worker.ShutdownReason;
-import com.amazonaws.services.kinesis.clientlibrary.types.InitializationInput;
-import com.amazonaws.services.kinesis.clientlibrary.types.ProcessRecordsInput;
-import com.amazonaws.services.kinesis.multilang.messages.CheckpointMessage;
-import com.amazonaws.services.kinesis.multilang.messages.InitializeMessage;
-import com.amazonaws.services.kinesis.multilang.messages.Message;
-import com.amazonaws.services.kinesis.multilang.messages.ProcessRecordsMessage;
-import com.amazonaws.services.kinesis.multilang.messages.ShutdownMessage;
-import com.amazonaws.services.kinesis.multilang.messages.ShutdownRequestedMessage;
-import com.amazonaws.services.kinesis.multilang.messages.StatusMessage;
-import lombok.extern.apachecommons.CommonsLog;
+package software.amazon.kinesis.multilang;
 
 import java.util.Optional;
 import java.util.concurrent.ExecutionException;
@@ -35,16 +20,36 @@ import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
+import lombok.extern.slf4j.Slf4j;
+import software.amazon.kinesis.exceptions.InvalidStateException;
+import software.amazon.kinesis.lifecycle.events.InitializationInput;
+import software.amazon.kinesis.lifecycle.events.LeaseLostInput;
+import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput;
+import software.amazon.kinesis.lifecycle.events.ShardEndedInput;
+import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
+import software.amazon.kinesis.multilang.messages.CheckpointMessage;
+import software.amazon.kinesis.multilang.messages.InitializeMessage;
+import software.amazon.kinesis.multilang.messages.LeaseLostMessage;
+import software.amazon.kinesis.multilang.messages.Message;
+import software.amazon.kinesis.multilang.messages.ProcessRecordsMessage;
+import software.amazon.kinesis.multilang.messages.ShardEndedMessage;
+import software.amazon.kinesis.multilang.messages.ShutdownRequestedMessage;
+import software.amazon.kinesis.multilang.messages.StatusMessage;
+import software.amazon.kinesis.processor.RecordProcessorCheckpointer;
+
 /**
  * An implementation of the multi language protocol.
  */
-@CommonsLog
+@Slf4j
 class MultiLangProtocol {
 
+    private final InitializationInput initializationInput;
+    private final Optional<Integer> timeoutInSeconds;
+
     private MessageReader messageReader;
     private MessageWriter messageWriter;
-    private final InitializationInput initializationInput;
-    private KinesisClientLibConfiguration configuration;
+
+    private MultiLangDaemonConfiguration configuration;
 
     /**
      * Constructor.
@@ -56,12 +61,16 @@ class MultiLangProtocol {
      * @param initializationInput
      *            information about the shard this processor is starting to process
      */
-    MultiLangProtocol(MessageReader messageReader, MessageWriter messageWriter,
-            InitializationInput initializationInput, KinesisClientLibConfiguration configuration) {
+    MultiLangProtocol(
+            MessageReader messageReader,
+            MessageWriter messageWriter,
+            InitializationInput initializationInput,
+            MultiLangDaemonConfiguration configuration) {
         this.messageReader = messageReader;
         this.messageWriter = messageWriter;
         this.initializationInput = initializationInput;
         this.configuration = configuration;
+        this.timeoutInSeconds = Optional.ofNullable(configuration.getTimeoutInSeconds());
     }
 
     /**
@@ -76,7 +85,6 @@ class MultiLangProtocol {
          */
         Future<Boolean> writeFuture = messageWriter.writeInitializeMessage(initializationInput);
         return waitForStatusMessage(InitializeMessage.ACTION, null, writeFuture);
-
     }
 
     /**
@@ -89,20 +97,30 @@ class MultiLangProtocol {
      */
     boolean processRecords(ProcessRecordsInput processRecordsInput) {
         Future<Boolean> writeFuture = messageWriter.writeProcessRecordsMessage(processRecordsInput);
-        return waitForStatusMessage(ProcessRecordsMessage.ACTION, processRecordsInput.getCheckpointer(), writeFuture);
+        return waitForStatusMessage(ProcessRecordsMessage.ACTION, processRecordsInput.checkpointer(), writeFuture);
     }
 
     /**
-     * Writes a {@link ShutdownMessage} to the child process's STDIN and waits for the child process to respond with a
-     * {@link StatusMessage} on its STDOUT.
+     * Notifies the client process that the lease has been lost, and it needs to shutdown.
      *
-     * @param checkpointer A checkpointer.
-     * @param reason Why this processor is being shutdown.
-     * @return Whether or not this operation succeeded.
+     * @param leaseLostInput
+     *            the lease lost input that is passed to the {@link MessageWriter}
+     * @return true if the message was successfully writtem
      */
-    boolean shutdown(IRecordProcessorCheckpointer checkpointer, ShutdownReason reason) {
-        Future<Boolean> writeFuture = messageWriter.writeShutdownMessage(reason);
-        return waitForStatusMessage(ShutdownMessage.ACTION, checkpointer, writeFuture);
+    boolean leaseLost(LeaseLostInput leaseLostInput) {
+        return waitForStatusMessage(LeaseLostMessage.ACTION, null, messageWriter.writeLeaseLossMessage(leaseLostInput));
+    }
+
+    /**
+     *
+     * @param shardEndedInput
+     * @return
+     */
+    boolean shardEnded(ShardEndedInput shardEndedInput) {
+        return waitForStatusMessage(
+                ShardEndedMessage.ACTION,
+                shardEndedInput.checkpointer(),
+                messageWriter.writeShardEndedMessage(shardEndedInput));
     }
 
     /**
@@ -112,14 +130,14 @@ class MultiLangProtocol {
      * @param checkpointer A checkpointer.
      * @return Whether or not this operation succeeded.
      */
-    boolean shutdownRequested(IRecordProcessorCheckpointer checkpointer) {
+    boolean shutdownRequested(RecordProcessorCheckpointer checkpointer) {
         Future<Boolean> writeFuture = messageWriter.writeShutdownRequestedMessage();
         return waitForStatusMessage(ShutdownRequestedMessage.ACTION, checkpointer, writeFuture);
     }
 
     /**
      * Waits for a {@link StatusMessage} for a particular action. If a {@link CheckpointMessage} is received, then this
-     * method will attempt to checkpoint with the provided {@link IRecordProcessorCheckpointer}. This method returns
+     * method will attempt to checkpoint with the provided {@link RecordProcessorCheckpointer}. This method returns
      * true if writing to the child process succeeds and the status message received back was for the correct action and
      * all communications with the child process regarding checkpointing were successful. Note that whether or not the
      * checkpointing itself was successful is not the concern of this method. This method simply cares whether it was
@@ -133,8 +151,8 @@ class MultiLangProtocol {
      *            The writing task.
      * @return Whether or not this operation succeeded.
      */
-    private boolean waitForStatusMessage(String action, IRecordProcessorCheckpointer checkpointer,
-            Future<Boolean> writeFuture) {
+    private boolean waitForStatusMessage(
+            String action, RecordProcessorCheckpointer checkpointer, Future<Boolean> writeFuture) {
         boolean statusWasCorrect = waitForStatusMessage(action, checkpointer);
 
         // Examine whether or not we failed somewhere along the line.
@@ -142,13 +160,10 @@ class MultiLangProtocol {
             boolean writerIsStillOpen = writeFuture.get();
             return statusWasCorrect && writerIsStillOpen;
         } catch (InterruptedException e) {
-            log.error(String.format("Interrupted while writing %s message for shard %s", action,
-                    initializationInput.getShardId()));
+            log.error("Interrupted while writing {} message for shard {}", action, initializationInput.shardId());
             return false;
         } catch (ExecutionException e) {
-            log.error(
-                    String.format("Failed to write %s message for shard %s", action, initializationInput.getShardId()),
-                    e);
+            log.error("Failed to write {} message for shard {}", action, initializationInput.shardId(), e);
             return false;
         }
     }
@@ -162,28 +177,28 @@ class MultiLangProtocol {
      *            the original process records request
      * @return Whether or not this operation succeeded.
      */
-    boolean waitForStatusMessage(String action, IRecordProcessorCheckpointer checkpointer) {
+    boolean waitForStatusMessage(String action, RecordProcessorCheckpointer checkpointer) {
         Optional<StatusMessage> statusMessage = Optional.empty();
         while (!statusMessage.isPresent()) {
             Future<Message> future = this.messageReader.getNextMessageFromSTDOUT();
-            Optional<Message> message = configuration.getTimeoutInSeconds()
-            .map(second -> futureMethod(() -> future.get(second, TimeUnit.SECONDS), action))
-            .orElse(futureMethod(future::get, action));
+            Optional<Message> message = timeoutInSeconds
+                    .map(second -> futureMethod(() -> future.get(second, TimeUnit.SECONDS), action))
+                    .orElse(futureMethod(future::get, action));
 
             if (!message.isPresent()) {
                 return false;
             }
 
-            Optional<Boolean> checkpointFailed = message.filter(m -> m instanceof CheckpointMessage )
-            .map(m -> (CheckpointMessage) m)
-            .flatMap(m -> futureMethod(() -> checkpoint(m, checkpointer).get(), "Checkpoint"))
-            .map(checkpointSuccess -> !checkpointSuccess);
+            Optional<Boolean> checkpointFailed = message.filter(m -> m instanceof CheckpointMessage)
+                    .map(m -> (CheckpointMessage) m)
+                    .flatMap(m -> futureMethod(() -> checkpoint(m, checkpointer).get(), "Checkpoint"))
+                    .map(checkpointSuccess -> !checkpointSuccess);
 
             if (checkpointFailed.orElse(false)) {
                 return false;
             }
 
-            statusMessage = message.filter(m -> m instanceof StatusMessage).map(m -> (StatusMessage) m );
+            statusMessage = message.filter(m -> m instanceof StatusMessage).map(m -> (StatusMessage) m);
         }
         return this.validateStatusMessage(statusMessage.get(), action);
     }
@@ -196,15 +211,19 @@ class MultiLangProtocol {
         try {
             return Optional.of(fm.get());
         } catch (InterruptedException e) {
-            log.error(String.format("Interrupted while waiting for %s message for shard %s", action,
-                    initializationInput.getShardId()), e);
+            log.error(
+                    "Interrupted while waiting for {} message for shard {}", action, initializationInput.shardId(), e);
         } catch (ExecutionException e) {
-            log.error(String.format("Failed to get status message for %s action for shard %s", action,
-                    initializationInput.getShardId()), e);
-        } catch (TimeoutException e) {
-            log.error(String.format("Timedout to get status message for %s action for shard %s. Terminating...",
+            log.error(
+                    "Failed to get status message for {} action for shard {}",
                     action,
-                    initializationInput.getShardId()),
+                    initializationInput.shardId(),
+                    e);
+        } catch (TimeoutException e) {
+            log.error(
+                    "Timedout to get status message for {} action for shard {}. Terminating...",
+                    action,
+                    initializationInput.shardId(),
                     e);
             haltJvm(1);
         }
@@ -229,24 +248,27 @@ class MultiLangProtocol {
      * @return Whether or not this operation succeeded.
      */
     private boolean validateStatusMessage(StatusMessage statusMessage, String action) {
-        log.info("Received response " + statusMessage + " from subprocess while waiting for " + action
-                + " while processing shard " + initializationInput.getShardId());
-        return !(statusMessage == null || statusMessage.getResponseFor() == null || !statusMessage.getResponseFor()
-                .equals(action));
-
+        log.info(
+                "Received response {} from subprocess while waiting for {}" + " while processing shard {}",
+                statusMessage,
+                action,
+                initializationInput.shardId());
+        return !(statusMessage == null
+                || statusMessage.getResponseFor() == null
+                || !statusMessage.getResponseFor().equals(action));
     }
 
     /**
-     * Attempts to checkpoint with the provided {@link IRecordProcessorCheckpointer} at the sequence number in the
+     * Attempts to checkpoint with the provided {@link RecordProcessorCheckpointer} at the sequence number in the
      * provided {@link CheckpointMessage}. If no sequence number is provided, i.e. the sequence number is null, then
-     * this method will call {@link IRecordProcessorCheckpointer#checkpoint()}. The method returns a future representing
+     * this method will call {@link RecordProcessorCheckpointer#checkpoint()}. The method returns a future representing
      * the attempt to write the result of this checkpoint attempt to the child process.
      *
      * @param checkpointMessage A checkpoint message.
      * @param checkpointer A checkpointer.
      * @return Whether or not this operation succeeded.
      */
-    private Future<Boolean> checkpoint(CheckpointMessage checkpointMessage, IRecordProcessorCheckpointer checkpointer) {
+    private Future<Boolean> checkpoint(CheckpointMessage checkpointMessage, RecordProcessorCheckpointer checkpointer) {
         String sequenceNumber = checkpointMessage.getSequenceNumber();
         Long subSequenceNumber = checkpointMessage.getSubSequenceNumber();
         try {
@@ -263,13 +285,12 @@ class MultiLangProtocol {
                 }
                 return this.messageWriter.writeCheckpointMessageWithError(sequenceNumber, subSequenceNumber, null);
             } else {
-                String message =
-                        String.format("Was asked to checkpoint at %s but no checkpointer was provided for shard %s",
-                                sequenceNumber, initializationInput.getShardId());
+                String message = String.format(
+                        "Was asked to checkpoint at %s but no checkpointer was provided for shard %s",
+                        sequenceNumber, initializationInput.shardId());
                 log.error(message);
-                return this.messageWriter.writeCheckpointMessageWithError(sequenceNumber, subSequenceNumber,
-                        new InvalidStateException(
-                        message));
+                return this.messageWriter.writeCheckpointMessageWithError(
+                        sequenceNumber, subSequenceNumber, new InvalidStateException(message));
             }
         } catch (Throwable t) {
             return this.messageWriter.writeCheckpointMessageWithError(sequenceNumber, subSequenceNumber, t);
@@ -277,8 +298,8 @@ class MultiLangProtocol {
     }
 
     private String logCheckpointMessage(String sequenceNumber, Long subSequenceNumber) {
-        return String.format("Attempting to checkpoint shard %s @ sequence number %s, and sub sequence number %s",
-                initializationInput.getShardId(), sequenceNumber, subSequenceNumber);
+        return String.format(
+                "Attempting to checkpoint shard %s @ sequence number %s, and sub sequence number %s",
+                initializationInput.shardId(), sequenceNumber, subSequenceNumber);
     }
-
 }

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MultiLangRecordProcessorFactory.java

@@ -0,0 +1,80 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package software.amazon.kinesis.multilang;
+
+import java.util.concurrent.ExecutorService;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.extern.slf4j.Slf4j;
+import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
+import software.amazon.kinesis.processor.ShardRecordProcessor;
+import software.amazon.kinesis.processor.ShardRecordProcessorFactory;
+
+/**
+ * Creates {@link MultiLangShardRecordProcessor}'s.
+ */
+@Slf4j
+public class MultiLangRecordProcessorFactory implements ShardRecordProcessorFactory {
+    private static final String COMMAND_DELIMETER_REGEX = " +";
+
+    private final String command;
+    private final String[] commandArray;
+
+    private final ObjectMapper objectMapper;
+
+    private final ExecutorService executorService;
+
+    private final MultiLangDaemonConfiguration configuration;
+
+    /**
+     * @param command The command that will do processing for this factory's record processors.
+     * @param executorService An executor service to use while processing inputs and outputs of the child process.
+     */
+    public MultiLangRecordProcessorFactory(
+            String command, ExecutorService executorService, MultiLangDaemonConfiguration configuration) {
+        this(command, executorService, new ObjectMapper(), configuration);
+    }
+
+    /**
+     * @param command The command that will do processing for this factory's record processors.
+     * @param executorService An executor service to use while processing inputs and outputs of the child process.
+     * @param objectMapper An object mapper used to convert messages to json to be written to the child process
+     */
+    public MultiLangRecordProcessorFactory(
+            String command,
+            ExecutorService executorService,
+            ObjectMapper objectMapper,
+            MultiLangDaemonConfiguration configuration) {
+        this.command = command;
+        this.commandArray = command.split(COMMAND_DELIMETER_REGEX);
+        this.executorService = executorService;
+        this.objectMapper = objectMapper;
+        this.configuration = configuration;
+    }
+
+    @Override
+    public ShardRecordProcessor shardRecordProcessor() {
+        log.debug("Creating new record processor for client executable: {}", command);
+        /*
+         * Giving ProcessBuilder the command as an array of Strings allows users to specify command line arguments.
+         */
+        return new MultiLangShardRecordProcessor(
+                new ProcessBuilder(commandArray), executorService, this.objectMapper, this.configuration);
+    }
+
+    String[] getCommandArray() {
+        return commandArray;
+    }
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MultiLangShardRecordProcessor.java

@@ -1,39 +1,35 @@
 /*
- *  Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- *  Licensed under the Amazon Software License (the "License").
- *  You may not use this file except in compliance with the License.
- *  A copy of the License is located at
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
- *  http://aws.amazon.com/asl/
- *
- *  or in the "license" file accompanying this file. This file is distributed
- *  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- *  express or implied. See the License for the specific language governing
- *  permissions and limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
-package com.amazonaws.services.kinesis.multilang;
+package software.amazon.kinesis.multilang;
 
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
+import java.util.function.Function;
 
-import com.amazonaws.services.kinesis.clientlibrary.exceptions.InvalidStateException;
-import com.amazonaws.services.kinesis.clientlibrary.exceptions.ShutdownException;
-import com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorCheckpointer;
-import com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IShutdownNotificationAware;
-import com.amazonaws.services.kinesis.clientlibrary.lib.worker.KinesisClientLibConfiguration;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IRecordProcessor;
-import com.amazonaws.services.kinesis.clientlibrary.types.InitializationInput;
-import com.amazonaws.services.kinesis.clientlibrary.types.ProcessRecordsInput;
-import com.amazonaws.services.kinesis.clientlibrary.types.ShutdownInput;
 import com.fasterxml.jackson.databind.ObjectMapper;
-
+import lombok.extern.slf4j.Slf4j;
+import software.amazon.kinesis.lifecycle.events.InitializationInput;
+import software.amazon.kinesis.lifecycle.events.LeaseLostInput;
+import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput;
+import software.amazon.kinesis.lifecycle.events.ShardEndedInput;
+import software.amazon.kinesis.lifecycle.events.ShutdownRequestedInput;
+import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
+import software.amazon.kinesis.processor.ShardRecordProcessor;
 
 /**
  * A record processor that manages creating a child process that implements the multi language protocol and connecting
@@ -41,9 +37,8 @@ import com.fasterxml.jackson.databind.ObjectMapper;
  * that object when its corresponding {@link #initialize}, {@link #processRecords}, and {@link #shutdown} methods are
  * called.
  */
-public class MultiLangRecordProcessor implements IRecordProcessor, IShutdownNotificationAware {
-
-    private static final Log LOG = LogFactory.getLog(MultiLangRecordProcessor.class);
+@Slf4j
+public class MultiLangShardRecordProcessor implements ShardRecordProcessor {
     private static final int EXIT_VALUE = 1;
 
     /** Whether or not record processor initialization is successful. Defaults to false. */
@@ -53,25 +48,25 @@ public class MultiLangRecordProcessor implements IRecordProcessor, IShutdownNoti
 
     private Future<?> stderrReadTask;
 
-    private MessageWriter messageWriter;
-    private MessageReader messageReader;
-    private DrainChildSTDERRTask readSTDERRTask;
+    private final MessageWriter messageWriter;
+    private final MessageReader messageReader;
+    private final DrainChildSTDERRTask readSTDERRTask;
 
-    private ProcessBuilder processBuilder;
+    private final ProcessBuilder processBuilder;
     private Process process;
-    private ExecutorService executorService;
+    private final ExecutorService executorService;
     private ProcessState state;
 
-    private ObjectMapper objectMapper;
+    private final ObjectMapper objectMapper;
 
     private MultiLangProtocol protocol;
 
-    private KinesisClientLibConfiguration configuration;
+    private final MultiLangDaemonConfiguration configuration;
 
     @Override
     public void initialize(InitializationInput initializationInput) {
         try {
-            this.shardId = initializationInput.getShardId();
+            this.shardId = initializationInput.shardId();
             try {
                 this.process = startProcess();
             } catch (IOException e) {
@@ -114,11 +109,33 @@ public class MultiLangRecordProcessor implements IRecordProcessor, IShutdownNoti
     }
 
     @Override
-    public void shutdown(ShutdownInput shutdownInput) {
+    public void leaseLost(LeaseLostInput leaseLostInput) {
+        shutdown(p -> p.leaseLost(leaseLostInput));
+    }
+
+    @Override
+    public void shardEnded(ShardEndedInput shardEndedInput) {
+        shutdown(p -> p.shardEnded(shardEndedInput));
+    }
+
+    @Override
+    public void shutdownRequested(ShutdownRequestedInput shutdownRequestedInput) {
+        log.info("Shutdown is requested.");
+        if (!initialized) {
+            log.info("Record processor was not initialized so no need to initiate a final checkpoint.");
+            return;
+        }
+        log.info("Requesting a checkpoint on shutdown notification.");
+        if (!protocol.shutdownRequested(shutdownRequestedInput.checkpointer())) {
+            log.error("Child process failed to complete shutdown notification.");
+        }
+    }
+
+    void shutdown(Function<MultiLangProtocol, Boolean> protocolInvocation) {
         // In cases where KCL loses lease for the shard after creating record processor instance but before
         // record processor initialize() is called, then shutdown() may be called directly before initialize().
         if (!initialized) {
-            LOG.info("Record processor was not initialized and will not have a child process, "
+            log.info("Record processor was not initialized and will not have a child process, "
                     + "so not invoking child process shutdown.");
             this.state = ProcessState.SHUTDOWN;
             return;
@@ -126,48 +143,37 @@ public class MultiLangRecordProcessor implements IRecordProcessor, IShutdownNoti
 
         try {
             if (ProcessState.ACTIVE.equals(this.state)) {
-                if (!protocol.shutdown(shutdownInput.getCheckpointer(), shutdownInput.getShutdownReason())) {
+                if (!protocolInvocation.apply(protocol)) {
                     throw new RuntimeException("Child process failed to shutdown");
                 }
 
                 childProcessShutdownSequence();
             } else {
-                LOG.warn("Shutdown was called but this processor is already shutdown. Not doing anything.");
+                log.warn("Shutdown was called but this processor is already shutdown. Not doing anything.");
             }
         } catch (Throwable t) {
             if (ProcessState.ACTIVE.equals(this.state)) {
                 stopProcessing("Encountered an error while trying to shutdown child process", t);
             } else {
-                stopProcessing("Encountered an error during shutdown,"
-                        + " but it appears the processor has already been shutdown", t);
+                stopProcessing(
+                        "Encountered an error during shutdown,"
+                                + " but it appears the processor has already been shutdown",
+                        t);
             }
         }
     }
 
-    @Override
-    public void shutdownRequested(IRecordProcessorCheckpointer checkpointer) {
-        LOG.info("Shutdown is requested.");
-        if (!initialized) {
-            LOG.info("Record processor was not initialized so no need to initiate a final checkpoint.");
-            return;
-        }
-        LOG.info("Requesting a checkpoint on shutdown notification.");
-        if (!protocol.shutdownRequested(checkpointer)) {
-            LOG.error("Child process failed to complete shutdown notification.");
-        }
-    }
-
-
     /**
      * Used to tell whether the processor has been shutdown already.
      */
     private enum ProcessState {
-        ACTIVE, SHUTDOWN
+        ACTIVE,
+        SHUTDOWN
     }
 
     /**
      * Constructor.
-     * 
+     *
      * @param processBuilder
      *            Provides process builder functionality.
      * @param executorService
@@ -175,15 +181,24 @@ public class MultiLangRecordProcessor implements IRecordProcessor, IShutdownNoti
      * @param objectMapper
      *            An obejct mapper.
      */
-    MultiLangRecordProcessor(ProcessBuilder processBuilder, ExecutorService executorService,
-            ObjectMapper objectMapper, KinesisClientLibConfiguration configuration) {
-        this(processBuilder, executorService, objectMapper, new MessageWriter(), new MessageReader(),
-                new DrainChildSTDERRTask(), configuration);
+    MultiLangShardRecordProcessor(
+            ProcessBuilder processBuilder,
+            ExecutorService executorService,
+            ObjectMapper objectMapper,
+            MultiLangDaemonConfiguration configuration) {
+        this(
+                processBuilder,
+                executorService,
+                objectMapper,
+                new MessageWriter(),
+                new MessageReader(),
+                new DrainChildSTDERRTask(),
+                configuration);
     }
 
     /**
      * Note: This constructor has package level access solely for testing purposes.
-     * 
+     *
      * @param processBuilder
      *            Provides the child process for this record processor
      * @param executorService
@@ -197,9 +212,14 @@ public class MultiLangRecordProcessor implements IRecordProcessor, IShutdownNoti
      * @param readSTDERRTask
      *            Error reader to read from child process's stderr
      */
-    MultiLangRecordProcessor(ProcessBuilder processBuilder, ExecutorService executorService, ObjectMapper objectMapper,
-                             MessageWriter messageWriter, MessageReader messageReader, DrainChildSTDERRTask readSTDERRTask,
-                             KinesisClientLibConfiguration configuration) {
+    MultiLangShardRecordProcessor(
+            ProcessBuilder processBuilder,
+            ExecutorService executorService,
+            ObjectMapper objectMapper,
+            MessageWriter messageWriter,
+            MessageReader messageReader,
+            DrainChildSTDERRTask readSTDERRTask,
+            MultiLangDaemonConfiguration configuration) {
         this.executorService = executorService;
         this.processBuilder = processBuilder;
         this.objectMapper = objectMapper;
@@ -208,7 +228,6 @@ public class MultiLangRecordProcessor implements IRecordProcessor, IShutdownNoti
         this.readSTDERRTask = readSTDERRTask;
         this.configuration = configuration;
 
-
         this.state = ProcessState.ACTIVE;
     }
 
@@ -228,7 +247,7 @@ public class MultiLangRecordProcessor implements IRecordProcessor, IShutdownNoti
                 messageWriter.close();
             }
         } catch (IOException e) {
-            LOG.error("Encountered exception while trying to close output stream.", e);
+            log.error("Encountered exception while trying to close output stream.", e);
         }
 
         // We should drain the STDOUT and STDERR of the child process. If we don't, the child process might remain
@@ -245,9 +264,9 @@ public class MultiLangRecordProcessor implements IRecordProcessor, IShutdownNoti
          * sure that it exits before we finish.
          */
         try {
-            LOG.info("Child process exited with value: " + process.waitFor());
+            log.info("Child process exited with value: {}", process.waitFor());
         } catch (InterruptedException e) {
-            LOG.error("Interrupted before process finished exiting. Attempting to kill process.");
+            log.error("Interrupted before process finished exiting. Attempting to kill process.");
             process.destroy();
         }
 
@@ -258,14 +277,14 @@ public class MultiLangRecordProcessor implements IRecordProcessor, IShutdownNoti
         try {
             inputStream.close();
         } catch (IOException e) {
-            LOG.error("Encountered exception while trying to close " + name + " stream.", e);
+            log.error("Encountered exception while trying to close {} stream.", name, e);
         }
     }
 
     /**
      * Convenience method used by {@link #childProcessShutdownSequence()} to drain the STDIN and STDERR of the child
      * process.
-     * 
+     *
      * @param future A future to wait on.
      * @param whatThisFutureIsDoing What that future is doing while we wait.
      */
@@ -273,33 +292,31 @@ public class MultiLangRecordProcessor implements IRecordProcessor, IShutdownNoti
         try {
             future.get();
         } catch (InterruptedException | ExecutionException e) {
-            LOG.error("Encountered error while " + whatThisFutureIsDoing + " for shard " + shardId, e);
+            log.error("Encountered error while {} for shard {}", whatThisFutureIsDoing, shardId, e);
         }
     }
 
     /**
      * Convenience method for logging and safely shutting down so that we don't throw an exception up to the KCL on
      * accident.
-     * 
+     *
      * @param message The reason we are stopping processing.
      * @param reason An exception that caused us to want to stop processing.
      */
     private void stopProcessing(String message, Throwable reason) {
         try {
-            LOG.error(message, reason);
+            log.error(message, reason);
             if (!state.equals(ProcessState.SHUTDOWN)) {
                 childProcessShutdownSequence();
             }
         } catch (Throwable t) {
-            LOG.error("Encountered error while trying to shutdown", t);
+            log.error("Encountered error while trying to shutdown", t);
         }
         exit();
     }
 
     /**
      * We provide a package level method for unit testing this call to exit.
-     * 
-     * @param val exit value
      */
     void exit() {
         System.exit(EXIT_VALUE);
@@ -308,7 +325,7 @@ public class MultiLangRecordProcessor implements IRecordProcessor, IShutdownNoti
     /**
      * The {@link ProcessBuilder} class is final so not easily mocked. We wrap the only interaction we have with it in
      * this package level method to permit unit testing.
-     * 
+     *
      * @return The process started by processBuilder
      * @throws IOException If the process can't be started.
      */

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/NestedPropertyKey.java

@@ -0,0 +1,148 @@
+/*
+ * Copyright 2023 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package software.amazon.kinesis.multilang;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.google.common.base.CaseFormat;
+import lombok.AccessLevel;
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+import software.amazon.awssdk.regions.Region;
+
+/**
+ * Key-Value pairs which may be nested in, and extracted from, a property value
+ * in a Java properties file. For example, given the line in a property file of
+ * {@code my_key = my_value|foo=bar} and a delimiter split on {@code |} (pipe),
+ * the value {@code my_value|foo=bar} would have a nested key of {@code foo}
+ * and its corresponding value is {@code bar}.
+ * <br/><br/>
+ * The order of nested properties does not matter, and these properties are optional.
+ * Customers may choose to provide, in any order, zero-or-more nested properties.
+ * <br/><br/>
+ * Duplicate keys are not supported, and may result in a last-write-wins outcome.
+ */
+@Slf4j
+public enum NestedPropertyKey {
+
+    /**
+     * Specify the service endpoint where requests will be submitted.
+     * This property's value must be in the following format:
+     * <pre>
+     *     ENDPOINT ::= SERVICE_ENDPOINT "^" SIGNING_REGION
+     *     SERVICE_ENDPOINT ::= URL
+     *     SIGNING_REGION ::= AWS_REGION
+     * </pre>
+     *
+     * It would be redundant to provide both this and {@link #ENDPOINT_REGION}.
+     *
+     * @see #ENDPOINT_REGION
+     * @see <a href="https://docs.aws.amazon.com/general/latest/gr/rande.html">AWS Service endpoints</a>
+     * @see <a href="https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-regions">Available Regions</a>
+     */
+    ENDPOINT {
+        void visit(final NestedPropertyProcessor processor, final String endpoint) {
+            final String[] tokens = endpoint.split("\\^");
+            if (tokens.length != 2) {
+                throw new IllegalArgumentException("Invalid " + name() + ": " + endpoint);
+            }
+            processor.acceptEndpoint(tokens[0], tokens[1]);
+        }
+    },
+
+    /**
+     * Specify the region where service requests will be submitted. This
+     * region will determine both the service endpoint and signing region.
+     * <br/><br/>
+     * It would be redundant to provide both this and {@link #ENDPOINT}.
+     *
+     * @see #ENDPOINT
+     * @see <a href="https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-regions">Available Regions</a>
+     */
+    ENDPOINT_REGION {
+        void visit(final NestedPropertyProcessor processor, final String regionName) {
+            List<Region> validRegions = Region.regions();
+            Region region = Region.of(regionName);
+            if (!validRegions.contains(region)) {
+                throw new IllegalArgumentException("Invalid region name: " + regionName);
+            }
+            processor.acceptEndpointRegion(region);
+        }
+    },
+
+    /**
+     * External ids may be used when delegating access in a multi-tenant
+     * environment, or to third parties.
+     *
+     * @see <a href="https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-user_externalid.html">
+     *     How to use an external ID when granting access to your AWS resources to a third party</a>
+     */
+    EXTERNAL_ID {
+        void visit(final NestedPropertyProcessor processor, final String externalId) {
+            processor.acceptExternalId(externalId);
+        }
+    },
+    ;
+
+    /**
+     * Nested key within the property value. For example, a nested key-value
+     * of {@code foo=bar} has a nested key of {@code foo}.
+     */
+    @Getter(AccessLevel.PACKAGE)
+    private final String nestedKey;
+
+    NestedPropertyKey() {
+        // convert the enum from UPPER_SNAKE_CASE to lowerCamelCase
+        nestedKey = CaseFormat.UPPER_UNDERSCORE.to(CaseFormat.LOWER_CAMEL, name());
+    }
+
+    abstract void visit(NestedPropertyProcessor processor, String value);
+
+    /**
+     * Parses any number of parameters. Each nested property will prompt a
+     * visit to the {@code processor}.
+     *
+     * @param processor processor to be invoked for every nested property
+     * @param params parameters to check for a nested property key
+     */
+    public static void parse(final NestedPropertyProcessor processor, final String... params) {
+        // Construct a disposable cache to keep this O(n). Since parsing is
+        // usually one-and-done, it's wasteful to maintain this cache in perpetuity.
+        final Map<String, NestedPropertyKey> cachedKeys = new HashMap<>();
+        for (final NestedPropertyKey npk : values()) {
+            cachedKeys.put(npk.getNestedKey(), npk);
+        }
+
+        for (final String param : params) {
+            if (param != null) {
+                final String[] tokens = param.split("=");
+                if (tokens.length == 2) {
+                    final NestedPropertyKey npk = cachedKeys.get(tokens[0]);
+                    if (npk != null) {
+                        npk.visit(processor, tokens[1]);
+                    } else {
+                        log.warn("Unsupported nested key: {}", param);
+                    }
+                } else if (tokens.length > 2) {
+                    log.warn("Malformed nested key: {}", param);
+                } else {
+                    log.info("Parameter is not a nested key: {}", param);
+                }
+            }
+        }
+    }
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/NestedPropertyProcessor.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2023 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package software.amazon.kinesis.multilang;
+
+import software.amazon.awssdk.regions.Region;
+
+/**
+ * Defines methods to process {@link NestedPropertyKey}s.
+ */
+public interface NestedPropertyProcessor {
+
+    /**
+     * Set the service endpoint where requests are sent.
+     *
+     * @param serviceEndpoint the service endpoint either with or without the protocol
+     *      (e.g., https://sns.us-west-1.amazonaws.com, sns.us-west-1.amazonaws.com)
+     * @param signingRegion the region to use for the client (e.g. us-west-1)
+     *
+     * @see #acceptEndpointRegion(Region)
+     * @see <a href="https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/core/client/builder/SdkClientBuilder.html#endpointOverride(java.net.URI)">
+     *     AwsClientBuilder.endpointOverride</a>
+     */
+    void acceptEndpoint(String serviceEndpoint, String signingRegion);
+
+    /**
+     * Set the service endpoint where requests are sent.
+     *
+     * @param region Region to be used by the client. This will be used to determine both the service endpoint
+     *      (e.g., https://sns.us-west-1.amazonaws.com) and signing region (e.g., us-west-1) for requests.
+     *
+     * @see #acceptEndpoint(String, String)
+     */
+    void acceptEndpointRegion(Region region);
+
+    /**
+     * Set the external id, an optional field to designate who can assume an IAM role.
+     *
+     * @param externalId external id used in the service call used to retrieve session credentials
+     */
+    void acceptExternalId(String externalId);
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/KclStsAssumeRoleCredentialsProvider.java

@@ -0,0 +1,75 @@
+/*
+ * Copyright 2024 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package software.amazon.kinesis.multilang.auth;
+
+import java.net.URI;
+import java.util.Arrays;
+
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.sts.StsClient;
+import software.amazon.awssdk.services.sts.StsClientBuilder;
+import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider;
+import software.amazon.awssdk.services.sts.model.AssumeRoleRequest;
+import software.amazon.awssdk.services.sts.model.AssumeRoleRequest.Builder;
+import software.amazon.kinesis.multilang.NestedPropertyKey;
+import software.amazon.kinesis.multilang.NestedPropertyProcessor;
+
+public class KclStsAssumeRoleCredentialsProvider implements AwsCredentialsProvider, NestedPropertyProcessor {
+    private final Builder assumeRoleRequestBuilder;
+    private final StsClientBuilder stsClientBuilder;
+    private final StsAssumeRoleCredentialsProvider stsAssumeRoleCredentialsProvider;
+
+    public KclStsAssumeRoleCredentialsProvider(String[] params) {
+        this(params[0], params[1], Arrays.copyOfRange(params, 2, params.length));
+    }
+
+    public KclStsAssumeRoleCredentialsProvider(String roleArn, String roleSessionName, String... params) {
+        this.assumeRoleRequestBuilder =
+                AssumeRoleRequest.builder().roleArn(roleArn).roleSessionName(roleSessionName);
+        this.stsClientBuilder = StsClient.builder();
+        NestedPropertyKey.parse(this, params);
+        this.stsAssumeRoleCredentialsProvider = StsAssumeRoleCredentialsProvider.builder()
+                .refreshRequest(assumeRoleRequestBuilder.build())
+                .asyncCredentialUpdateEnabled(true)
+                .stsClient(stsClientBuilder.build())
+                .build();
+    }
+
+    @Override
+    public AwsCredentials resolveCredentials() {
+        return stsAssumeRoleCredentialsProvider.resolveCredentials();
+    }
+
+    @Override
+    public void acceptEndpoint(String serviceEndpoint, String signingRegion) {
+        if (!serviceEndpoint.startsWith("http://") && !serviceEndpoint.startsWith("https://")) {
+            serviceEndpoint = "https://" + serviceEndpoint;
+        }
+        stsClientBuilder.endpointOverride(URI.create(serviceEndpoint));
+        stsClientBuilder.region(Region.of(signingRegion));
+    }
+
+    @Override
+    public void acceptEndpointRegion(Region region) {
+        stsClientBuilder.region(region);
+    }
+
+    @Override
+    public void acceptExternalId(String externalId) {
+        assumeRoleRequestBuilder.externalId(externalId);
+    }
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AwsCredentialsProviderPropertyValueDecoder.java

@@ -0,0 +1,261 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package software.amazon.kinesis.multilang.config;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import lombok.extern.slf4j.Slf4j;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain;
+import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider;
+import software.amazon.kinesis.multilang.auth.KclStsAssumeRoleCredentialsProvider;
+
+/**
+ * Get AwsCredentialsProvider property.
+ */
+@Slf4j
+class AwsCredentialsProviderPropertyValueDecoder implements IPropertyValueDecoder<AwsCredentialsProvider> {
+    private static final String LIST_DELIMITER = ",";
+    private static final String ARG_DELIMITER = "|";
+
+    /**
+     * Constructor.
+     */
+    AwsCredentialsProviderPropertyValueDecoder() {}
+
+    /**
+     * Get AwsCredentialsProvider property.
+     *
+     * @param value
+     *            property value as String
+     * @return corresponding variable in correct type
+     */
+    @Override
+    public AwsCredentialsProvider decodeValue(String value) {
+        if (value != null) {
+            List<String> providerNames = getProviderNames(value);
+            List<AwsCredentialsProvider> providers = getValidCredentialsProviders(providerNames);
+            AwsCredentialsProvider[] ps = new AwsCredentialsProvider[providers.size()];
+            providers.toArray(ps);
+            if (providers.isEmpty()) {
+                log.warn("Unable to construct any provider with name {}", value);
+                log.warn("Please verify that all AwsCredentialsProvider properties are passed correctly");
+            }
+            return AwsCredentialsProviderChain.builder()
+                    .credentialsProviders(providers)
+                    .build();
+        } else {
+            throw new IllegalArgumentException("Property AwsCredentialsProvider is missing.");
+        }
+    }
+
+    /**
+     * @return list of supported types
+     */
+    @Override
+    public List<Class<AwsCredentialsProvider>> getSupportedTypes() {
+        return Collections.singletonList(AwsCredentialsProvider.class);
+    }
+
+    /**
+     * Convert string list to a list of valid credentials providers.
+     */
+    private static List<AwsCredentialsProvider> getValidCredentialsProviders(List<String> providerNames) {
+        List<AwsCredentialsProvider> credentialsProviders = new ArrayList<>();
+
+        for (String providerName : providerNames) {
+            final String[] nameAndArgs = providerName.split("\\" + ARG_DELIMITER);
+            final Class<? extends AwsCredentialsProvider> clazz = getClass(nameAndArgs[0]);
+            if (clazz == null) {
+                continue;
+            }
+            log.info("Attempting to construct {}", clazz);
+            final String[] varargs =
+                    nameAndArgs.length > 1 ? Arrays.copyOfRange(nameAndArgs, 1, nameAndArgs.length) : new String[0];
+            AwsCredentialsProvider provider = tryConstructor(providerName, clazz, varargs);
+            if (provider == null) {
+                provider = tryCreate(providerName, clazz, varargs);
+            }
+            if (provider != null) {
+                log.info("Provider constructed successfully: {}", provider);
+                credentialsProviders.add(provider);
+            }
+        }
+        return credentialsProviders;
+    }
+
+    private static AwsCredentialsProvider tryConstructor(
+            String providerName, Class<? extends AwsCredentialsProvider> clazz, String[] varargs) {
+        AwsCredentialsProvider provider =
+                constructProvider(providerName, () -> getConstructorWithVarArgs(clazz, varargs));
+        if (provider == null) {
+            provider = constructProvider(providerName, () -> getConstructorWithArgs(clazz, varargs));
+        }
+        if (provider == null) {
+            provider = constructProvider(providerName, clazz::newInstance);
+        }
+        return provider;
+    }
+
+    private static AwsCredentialsProvider tryCreate(
+            String providerName, Class<? extends AwsCredentialsProvider> clazz, String[] varargs) {
+        AwsCredentialsProvider provider =
+                constructProvider(providerName, () -> getCreateMethod(clazz, (Object) varargs));
+        if (provider == null) {
+            provider = constructProvider(providerName, () -> getCreateMethod(clazz, varargs));
+        }
+        if (provider == null) {
+            provider = constructProvider(providerName, () -> getCreateMethod(clazz));
+        }
+        return provider;
+    }
+
+    private static AwsCredentialsProvider getConstructorWithVarArgs(
+            Class<? extends AwsCredentialsProvider> clazz, String[] varargs) {
+        try {
+            return clazz.getConstructor(String[].class).newInstance((Object) varargs);
+        } catch (Exception e) {
+            return null;
+        }
+    }
+
+    private static AwsCredentialsProvider getConstructorWithArgs(
+            Class<? extends AwsCredentialsProvider> clazz, String[] varargs) {
+        try {
+            Class<?>[] argTypes = new Class<?>[varargs.length];
+            Arrays.fill(argTypes, String.class);
+            return clazz.getConstructor(argTypes).newInstance((Object[]) varargs);
+        } catch (Exception e) {
+            return null;
+        }
+    }
+
+    private static AwsCredentialsProvider getCreateMethod(
+            Class<? extends AwsCredentialsProvider> clazz, Object... args) {
+        try {
+            Class<?>[] argTypes = new Class<?>[args.length];
+            for (int i = 0; i < args.length; i++) {
+                argTypes[i] = args[i].getClass();
+            }
+            Method createMethod = clazz.getDeclaredMethod("create", argTypes);
+            if (Modifier.isStatic(createMethod.getModifiers())) {
+                return clazz.cast(createMethod.invoke(null, args));
+            } else {
+                log.warn("Found non-static create() method in {}", clazz.getName());
+            }
+        } catch (NoSuchMethodException e) {
+            // No matching create method found for class
+        } catch (Exception e) {
+            log.warn("Failed to invoke create() method in {}", clazz.getName(), e);
+        }
+        return null;
+    }
+
+    /**
+     * Resolves the class for the given provider name.
+     *
+     * @param providerName A string containing the provider name.
+     *
+     * @return The Class object representing the resolved AwsCredentialsProvider implementation,
+     * or null if the class cannot be resolved or does not extend AwsCredentialsProvider.
+     */
+    private static Class<? extends AwsCredentialsProvider> getClass(String providerName) {
+        // Convert any form of StsAssumeRoleCredentialsProvider string to KclStsAssumeRoleCredentialsProvider
+        if (providerName.equals(StsAssumeRoleCredentialsProvider.class.getSimpleName())
+                || providerName.equals(StsAssumeRoleCredentialsProvider.class.getName())) {
+            providerName = KclStsAssumeRoleCredentialsProvider.class.getName();
+        }
+        try {
+            final Class<?> c = Class.forName(providerName);
+            if (!AwsCredentialsProvider.class.isAssignableFrom(c)) {
+                return null;
+            }
+            return (Class<? extends AwsCredentialsProvider>) c;
+        } catch (ClassNotFoundException cnfe) {
+            // Providers are a product of prefixed Strings to cover multiple
+            // namespaces (e.g., "Foo" -> { "some.auth.Foo", "kcl.auth.Foo" }).
+            // It's expected that many class names will not resolve.
+            return null;
+        }
+    }
+
+    private static List<String> getProviderNames(String property) {
+        // assume list delimiter is ","
+        String[] elements = property.split(LIST_DELIMITER);
+        List<String> result = new ArrayList<>();
+        for (int i = 0; i < elements.length; i++) {
+            String string = elements[i].trim();
+            if (!string.isEmpty()) {
+                // find all possible names and add them to name list
+                result.addAll(getPossibleFullClassNames(string));
+            }
+        }
+        return result;
+    }
+
+    private static List<String> getPossibleFullClassNames(final String provider) {
+        return Stream.of(
+                        // Customer provides a short name of a provider offered by this multi-lang package
+                        "software.amazon.kinesis.multilang.auth.",
+                        // Customer provides a short name of common providers in software.amazon.awssdk.auth.credentials
+                        // package (e.g., any classes implementing the AwsCredentialsProvider interface)
+                        // @see
+                        // https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/auth/credentials/AwsCredentialsProvider.html
+                        "software.amazon.awssdk.auth.credentials.",
+                        // Customer provides a fully-qualified provider name, or a custom credentials provider
+                        // (e.g., org.mycompany.FooProvider)
+                        "")
+                .map(prefix -> prefix + provider)
+                .collect(Collectors.toList());
+    }
+
+    @FunctionalInterface
+    private interface CredentialsProviderConstructor<T extends AwsCredentialsProvider> {
+        T construct()
+                throws IllegalAccessException, InstantiationException, InvocationTargetException, NoSuchMethodException;
+    }
+
+    /**
+     * Attempts to construct an {@link AwsCredentialsProvider}.
+     *
+     * @param providerName Raw, unmodified provider name. Should there be an
+     *      Exception during construction, this parameter will be logged.
+     * @param constructor supplier-like function that will perform the construction
+     * @return the constructed provider, if successful; otherwise, null
+     *
+     * @param <T> type of the CredentialsProvider to construct
+     */
+    private static <T extends AwsCredentialsProvider> T constructProvider(
+            final String providerName, final CredentialsProviderConstructor<T> constructor) {
+        try {
+            return constructor.construct();
+        } catch (NoSuchMethodException
+                | IllegalAccessException
+                | InstantiationException
+                | InvocationTargetException
+                | RuntimeException ignored) {
+            // ignore
+        }
+        return null;
+    }
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/BuilderDynaBean.java

@@ -0,0 +1,292 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.multilang.config;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.function.Function;
+
+import lombok.Getter;
+import org.apache.commons.beanutils.ConvertUtilsBean;
+import org.apache.commons.beanutils.DynaBean;
+import org.apache.commons.beanutils.DynaClass;
+import org.apache.commons.beanutils.DynaProperty;
+import org.apache.commons.lang3.ClassUtils;
+import org.apache.commons.lang3.StringUtils;
+
+public class BuilderDynaBean implements DynaBean {
+
+    private static final String[] CLASS_NAME_JOINERS = {ClassUtils.PACKAGE_SEPARATOR, ClassUtils.INNER_CLASS_SEPARATOR};
+    static final String NO_MAP_ACCESS_SUPPORT = "Map access isn't supported";
+
+    private Class<?> destinedClass;
+    private final ConvertUtilsBean convertUtilsBean;
+    private final List<String> classPrefixSearchList;
+
+    private DynaBeanCreateSupport dynaBeanCreateSupport;
+    private DynaBeanBuilderSupport dynaBeanBuilderSupport;
+
+    @Getter
+    private boolean isDirty = false;
+
+    private final Function<String, ?> emptyPropertyHandler;
+    private Object emptyPropertyResolved = null;
+
+    public BuilderDynaBean(Class<?> destinedClass, ConvertUtilsBean convertUtilsBean, String... classPrefixSearchList) {
+        this(destinedClass, convertUtilsBean, null, Arrays.asList(classPrefixSearchList));
+    }
+
+    public BuilderDynaBean(
+            Class<?> destinedClass,
+            ConvertUtilsBean convertUtilsBean,
+            Function<String, ?> emptyPropertyHandler,
+            String... classPrefixSearchList) {
+        this(destinedClass, convertUtilsBean, emptyPropertyHandler, Arrays.asList(classPrefixSearchList));
+    }
+
+    public BuilderDynaBean(
+            Class<?> destinedClass,
+            ConvertUtilsBean convertUtilsBean,
+            Function<String, ?> emptyPropertyHandler,
+            List<String> classPrefixSearchList) {
+        this.convertUtilsBean = convertUtilsBean;
+        this.classPrefixSearchList = classPrefixSearchList;
+        this.emptyPropertyHandler = emptyPropertyHandler;
+        initialize(destinedClass);
+    }
+
+    private void initialize(Class<?> destinedClass) {
+        this.destinedClass = destinedClass;
+
+        if (DynaBeanBuilderUtils.isBuilderOrCreate(destinedClass)) {
+            dynaBeanBuilderSupport = new DynaBeanBuilderSupport(destinedClass, convertUtilsBean, classPrefixSearchList);
+            dynaBeanCreateSupport = new DynaBeanCreateSupport(destinedClass, convertUtilsBean, classPrefixSearchList);
+        }
+    }
+
+    private void reinitializeFrom(String newClass) {
+        Class<?> newClazz = null;
+        List<String> attempts = new ArrayList<>();
+        attempts.add(newClass);
+        try {
+            newClazz = Class.forName(newClass);
+        } catch (ClassNotFoundException e) {
+            //
+            // Ignored
+            //
+        }
+        if (newClazz == null) {
+            for (String prefix : classPrefixSearchList) {
+                for (String joiner : CLASS_NAME_JOINERS) {
+                    String possibleClass;
+                    if (prefix.endsWith(joiner)) {
+                        possibleClass = prefix + newClass;
+                    } else {
+                        possibleClass = prefix + joiner + newClass;
+                    }
+                    attempts.add(possibleClass);
+                    try {
+                        newClazz = Class.forName(possibleClass);
+                        break;
+                    } catch (ClassNotFoundException e) {
+                        //
+                        // Ignored
+                        //
+                    }
+                }
+            }
+        }
+
+        if (newClazz == null) {
+            throw new IllegalArgumentException(
+                    "Unable to load class " + newClass + ". Attempted: (" + String.join(", ", attempts) + ")");
+        }
+        initialize(newClazz);
+    }
+
+    private void validatedExpectedClass(Class<?> source, Class<?> expected) {
+        if (!ClassUtils.isAssignable(source, expected)) {
+            throw new IllegalArgumentException(
+                    String.format("%s cannot be assigned to %s.", source.getName(), expected.getName()));
+        }
+    }
+
+    public boolean canBuildOrCreate() {
+        return dynaBeanBuilderSupport != null || dynaBeanCreateSupport != null;
+    }
+
+    private void validateCanBuildOrCreate() {
+        if (!canBuildOrCreate()) {
+            throw new IllegalStateException("Unable to to introspect or handle " + destinedClass.getName()
+                    + " as it doesn't have a builder or create method.");
+        }
+    }
+
+    @SafeVarargs
+    public final <T> T build(Class<T> expected, Function<Object, Object>... additionalMutators) {
+        if (emptyPropertyResolved != null) {
+            validatedExpectedClass(emptyPropertyResolved.getClass(), expected);
+            return expected.cast(emptyPropertyResolved);
+        }
+
+        if (dynaBeanBuilderSupport == null && dynaBeanCreateSupport == null) {
+            return null;
+        }
+
+        validatedExpectedClass(destinedClass, expected);
+        if (dynaBeanBuilderSupport.isValid()) {
+            return expected.cast(dynaBeanBuilderSupport.build(additionalMutators));
+        } else {
+            return expected.cast(dynaBeanCreateSupport.build());
+        }
+    }
+
+    private void validateResolvedEmptyHandler() {
+        if (emptyPropertyResolved != null) {
+            throw new IllegalStateException("When a property handler is resolved further properties may not be set.");
+        }
+    }
+
+    boolean hasValue(String name) {
+        if (dynaBeanBuilderSupport != null) {
+            return dynaBeanBuilderSupport.hasValue(name);
+        }
+        return false;
+    }
+
+    @Override
+    public boolean contains(String name, String key) {
+        throw new UnsupportedOperationException(NO_MAP_ACCESS_SUPPORT);
+    }
+
+    @Override
+    public Object get(String name) {
+        validateResolvedEmptyHandler();
+        isDirty = true;
+        return dynaBeanBuilderSupport.get(name);
+    }
+
+    @Override
+    public Object get(String name, int index) {
+        validateResolvedEmptyHandler();
+        isDirty = true;
+        if (StringUtils.isEmpty(name)) {
+            return dynaBeanCreateSupport.get(name, index);
+        }
+        return dynaBeanBuilderSupport.get(name, index);
+    }
+
+    @Override
+    public Object get(String name, String key) {
+        throw new UnsupportedOperationException(NO_MAP_ACCESS_SUPPORT);
+    }
+
+    @Override
+    public DynaClass getDynaClass() {
+        return new DynaClass() {
+            @Override
+            public String getName() {
+                return destinedClass.getName();
+            }
+
+            @Override
+            public DynaProperty getDynaProperty(String name) {
+                if (StringUtils.isEmpty(name)) {
+                    return new DynaProperty(name);
+                }
+                if ("class".equals(name)) {
+                    return new DynaProperty(name, String.class);
+                }
+                //
+                // We delay validation until after the class check to allow for re-initialization for a specific class.
+                // The check for isEmpty is allowed ahead of this check to allow for raw string support.
+                //
+                validateCanBuildOrCreate();
+                List<TypeTag> types = dynaBeanBuilderSupport.getProperty(name);
+                if (types.size() > 1) {
+                    Optional<TypeTag> arrayType =
+                            types.stream().filter(t -> t.type.isArray()).findFirst();
+                    return arrayType
+                            .map(t -> new DynaProperty(name, t.type, t.type.getComponentType()))
+                            .orElseGet(() -> new DynaProperty(name));
+                } else {
+                    TypeTag type = types.get(0);
+                    if (type.hasConverter) {
+                        return new DynaProperty(name, type.type);
+                    }
+                    if (type.type.isEnum()) {
+                        return new DynaProperty(name, String.class);
+                    }
+                    return new DynaProperty(name, BuilderDynaBean.class);
+                }
+            }
+
+            @Override
+            public DynaProperty[] getDynaProperties() {
+                validateCanBuildOrCreate();
+                return dynaBeanBuilderSupport.getPropertyNames().stream()
+                        .map(this::getDynaProperty)
+                        .toArray(DynaProperty[]::new);
+            }
+
+            @Override
+            public DynaBean newInstance() {
+                return null;
+            }
+        };
+    }
+
+    @Override
+    public void remove(String name, String key) {
+        throw new UnsupportedOperationException(NO_MAP_ACCESS_SUPPORT);
+    }
+
+    @Override
+    public void set(String name, Object value) {
+        validateResolvedEmptyHandler();
+        isDirty = true;
+        if (emptyPropertyHandler != null && StringUtils.isEmpty(name) && value instanceof String) {
+            emptyPropertyResolved = emptyPropertyHandler.apply((String) value);
+            return;
+        }
+
+        if ("class".equals(name)) {
+            reinitializeFrom(value.toString());
+        } else {
+            validateResolvedEmptyHandler();
+            dynaBeanBuilderSupport.set(name, value);
+        }
+    }
+
+    @Override
+    public void set(String name, int index, Object value) {
+        validateResolvedEmptyHandler();
+        validateCanBuildOrCreate();
+        isDirty = true;
+        if (StringUtils.isEmpty(name)) {
+            dynaBeanCreateSupport.set(name, index, value);
+        } else {
+            dynaBeanBuilderSupport.set(name, index, value);
+        }
+    }
+
+    @Override
+    public void set(String name, String key, Object value) {
+        throw new UnsupportedOperationException(NO_MAP_ACCESS_SUPPORT);
+    }
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/ConfigurationSettable.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.multilang.config;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Repeatable;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.FIELD)
+@Repeatable(ConfigurationSettables.class)
+public @interface ConfigurationSettable {
+
+    /**
+     * Which builder this option applies to
+     *
+     * @return the class of the builder to use
+     */
+    Class<?> configurationClass();
+
+    /**
+     * The method name on the builder, defaults to the fieldName
+     *
+     * @return the name of the method or null to use the default
+     */
+    String methodName() default "";
+
+    /**
+     * If the type is actually an optional value this will enable conversions
+     *
+     * @return true if the value should be wrapped by an optional
+     */
+    boolean convertToOptional() default false;
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/ConfigurationSettableUtils.java

@@ -0,0 +1,122 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.multilang.config;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
+import com.google.common.base.Defaults;
+import lombok.NonNull;
+import org.apache.commons.lang3.ClassUtils;
+import org.apache.commons.lang3.StringUtils;
+
+public class ConfigurationSettableUtils {
+
+    public static <T> T resolveFields(@NonNull Object source, @NonNull T configObject) {
+        Map<Class<?>, Object> configObjects = new HashMap<>();
+        configObjects.put(configObject.getClass(), configObject);
+        resolveFields(source, configObjects, null, null);
+
+        return configObject;
+    }
+
+    public static void resolveFields(
+            Object source, Map<Class<?>, Object> configObjects, Set<Class<?>> restrictTo, Set<Class<?>> skipIf) {
+        for (Field field : source.getClass().getDeclaredFields()) {
+            for (ConfigurationSettable b : field.getAnnotationsByType(ConfigurationSettable.class)) {
+                if (restrictTo != null && !restrictTo.contains(b.configurationClass())) {
+                    continue;
+                }
+                if (skipIf != null && skipIf.contains(b.configurationClass())) {
+                    continue;
+                }
+                field.setAccessible(true);
+                Object configObject = configObjects.get(b.configurationClass());
+                if (configObject != null) {
+                    String setterName = field.getName();
+                    if (!StringUtils.isEmpty(b.methodName())) {
+                        setterName = b.methodName();
+                    }
+                    Object value;
+                    try {
+                        value = field.get(source);
+                    } catch (IllegalAccessException e) {
+                        throw new RuntimeException(e);
+                    }
+
+                    if (value != null && !value.equals(Defaults.defaultValue(field.getType()))) {
+                        Method setter = null;
+                        if (b.convertToOptional()) {
+                            value = Optional.of(value);
+                        }
+                        if (ClassUtils.isPrimitiveOrWrapper(value.getClass())) {
+                            Class<?> primitiveType = field.getType().isPrimitive()
+                                    ? field.getType()
+                                    : ClassUtils.wrapperToPrimitive(field.getType());
+                            Class<?> wrapperType = !field.getType().isPrimitive()
+                                    ? field.getType()
+                                    : ClassUtils.primitiveToWrapper(field.getType());
+
+                            try {
+                                setter = b.configurationClass().getMethod(setterName, primitiveType);
+                            } catch (NoSuchMethodException e) {
+                                //
+                                // Ignore this
+                                //
+                            }
+                            if (setter == null) {
+                                try {
+                                    setter = b.configurationClass().getMethod(setterName, wrapperType);
+                                } catch (NoSuchMethodException e) {
+                                    throw new RuntimeException(e);
+                                }
+                            }
+                        } else {
+                            try {
+                                setter = b.configurationClass().getMethod(setterName, value.getClass());
+                            } catch (NoSuchMethodException e) {
+                                // find if there is a setter which is not the exact parameter type
+                                // but is assignable from the type
+                                for (Method method : b.configurationClass().getMethods()) {
+                                    Class<?>[] parameterTypes = method.getParameterTypes();
+                                    if (method.getName().equals(setterName)
+                                            && parameterTypes.length == 1
+                                            && parameterTypes[0].isAssignableFrom(value.getClass())) {
+                                        setter = method;
+                                        break;
+                                    }
+                                }
+                                if (setter == null) {
+                                    throw new RuntimeException(e);
+                                }
+                            }
+                        }
+                        try {
+                            setter.invoke(configObject, value);
+                        } catch (IllegalAccessException | InvocationTargetException e) {
+                            throw new RuntimeException(e);
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/ConfigurationSettables.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.multilang.config;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.FIELD)
+public @interface ConfigurationSettables {
+    ConfigurationSettable[] value();
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/CoordinatorStateTableConfigBean.java

@@ -0,0 +1,82 @@
+/*
+ * Copyright 2024 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.multilang.config;
+
+import lombok.Getter;
+import lombok.Setter;
+import software.amazon.awssdk.services.dynamodb.model.BillingMode;
+import software.amazon.kinesis.coordinator.CoordinatorConfig.CoordinatorStateTableConfig;
+import software.amazon.kinesis.multilang.config.converter.TagConverter.TagCollection;
+
+@Getter
+@Setter
+public class CoordinatorStateTableConfigBean {
+
+    interface CoordinatorStateConfigBeanDelegate {
+        String getCoordinatorStateTableName();
+
+        void setCoordinatorStateTableName(String value);
+
+        BillingMode getCoordinatorStateBillingMode();
+
+        void setCoordinatorStateBillingMode(BillingMode value);
+
+        long getCoordinatorStateReadCapacity();
+
+        void setCoordinatorStateReadCapacity(long value);
+
+        long getCoordinatorStateWriteCapacity();
+
+        void setCoordinatorStateWriteCapacity(long value);
+
+        Boolean getCoordinatorStatePointInTimeRecoveryEnabled();
+
+        void setCoordinatorStatePointInTimeRecoveryEnabled(Boolean value);
+
+        Boolean getCoordinatorStateDeletionProtectionEnabled();
+
+        void setCoordinatorStateDeletionProtectionEnabled(Boolean value);
+
+        TagCollection getCoordinatorStateTags();
+
+        void setCoordinatorStateTags(TagCollection value);
+    }
+
+    @ConfigurationSettable(configurationClass = CoordinatorStateTableConfig.class, methodName = "tableName")
+    private String coordinatorStateTableName;
+
+    @ConfigurationSettable(configurationClass = CoordinatorStateTableConfig.class, methodName = "billingMode")
+    private BillingMode coordinatorStateBillingMode;
+
+    @ConfigurationSettable(configurationClass = CoordinatorStateTableConfig.class, methodName = "readCapacity")
+    private long coordinatorStateReadCapacity;
+
+    @ConfigurationSettable(configurationClass = CoordinatorStateTableConfig.class, methodName = "writeCapacity")
+    private long coordinatorStateWriteCapacity;
+
+    @ConfigurationSettable(
+            configurationClass = CoordinatorStateTableConfig.class,
+            methodName = "pointInTimeRecoveryEnabled")
+    private Boolean coordinatorStatePointInTimeRecoveryEnabled;
+
+    @ConfigurationSettable(
+            configurationClass = CoordinatorStateTableConfig.class,
+            methodName = "deletionProtectionEnabled")
+    private Boolean coordinatorStateDeletionProtectionEnabled;
+
+    @ConfigurationSettable(configurationClass = CoordinatorStateTableConfig.class, methodName = "tags")
+    private TagCollection coordinatorStateTags;
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/DatePropertyValueDecoder.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package software.amazon.kinesis.multilang.config;
+
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
+
+/**
+ * Provide Date property.
+ */
+public class DatePropertyValueDecoder implements IPropertyValueDecoder<Date> {
+
+    /**
+     * Constructor.
+     */
+    DatePropertyValueDecoder() {}
+
+    /**
+     * @param value property value as String
+     * @return corresponding variable in correct type
+     */
+    @Override
+    public Date decodeValue(String value) {
+        try {
+            return new Date(Long.parseLong(value) * 1000L);
+        } catch (NumberFormatException e) {
+            throw new IllegalArgumentException("Date property value must be numeric.");
+        }
+    }
+
+    /**
+     * @return list of supported types
+     */
+    @Override
+    public List<Class<Date>> getSupportedTypes() {
+        return Arrays.asList(Date.class);
+    }
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/DynaBeanBuilderSupport.java

@@ -0,0 +1,258 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.multilang.config;
+
+import java.lang.reflect.Array;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import java.util.function.Supplier;
+
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
+import org.apache.commons.beanutils.ConvertUtilsBean;
+import org.apache.commons.lang3.ClassUtils;
+
+class DynaBeanBuilderSupport {
+
+    private static final String BUILD_METHOD_NAME = "build";
+    private static final String BUILDER_METHOD_NAME = "builder";
+
+    private final Class<?> destinedClass;
+    private final ConvertUtilsBean convertUtilsBean;
+    private final List<String> classPrefixSearchList;
+    private final Class<?> builderClass;
+
+    private final Multimap<String, TypeTag> properties = HashMultimap.create();
+    private final Map<String, Object> values = new HashMap<>();
+
+    DynaBeanBuilderSupport(
+            Class<?> destinedClass, ConvertUtilsBean convertUtilsBean, List<String> classPrefixSearchList) {
+        this.destinedClass = destinedClass;
+        this.convertUtilsBean = convertUtilsBean;
+        this.classPrefixSearchList = classPrefixSearchList;
+        this.builderClass = builderClassFrom(destinedClass);
+
+        buildProperties();
+    }
+
+    private static Class<?> builderClassFrom(Class<?> destinedClass) {
+        Method builderMethod;
+        try {
+            builderMethod = destinedClass.getMethod(BUILDER_METHOD_NAME);
+        } catch (NoSuchMethodException e) {
+            return null;
+        }
+
+        return builderMethod.getReturnType();
+    }
+
+    private void buildProperties() {
+        if (builderClass == null) {
+            return;
+        }
+        try {
+            builderClass.getMethod(BUILD_METHOD_NAME);
+        } catch (NoSuchMethodException e) {
+            throw new RuntimeException(e);
+        }
+
+        for (Method method : builderClass.getMethods()) {
+            if (method.getParameterCount() == 1 && ClassUtils.isAssignable(builderClass, method.getReturnType())) {
+                Class<?> paramType = method.getParameterTypes()[0];
+                if (Supplier.class.isAssignableFrom(paramType) || Consumer.class.isAssignableFrom(paramType)) {
+                    continue;
+                }
+                if (paramType.isEnum()) {
+                    properties.put(method.getName(), new TypeTag(paramType, true, method));
+                } else if (convertUtilsBean.lookup(paramType) == null) {
+                    properties.put(method.getName(), new TypeTag(paramType, false, method));
+                } else {
+                    properties.put(method.getName(), new TypeTag(paramType, true, method));
+                }
+            }
+        }
+    }
+
+    boolean isValid() {
+        return builderClass != null;
+    }
+
+    private Object createForProperty(String name) {
+        Optional<TypeTag> type = properties.get(name).stream().findFirst();
+        return type.map(t -> {
+                    if (DynaBeanBuilderUtils.isBuilderOrCreate(t.type) || !t.hasConverter) {
+                        return new BuilderDynaBean(t.type, convertUtilsBean, null, classPrefixSearchList);
+                    }
+                    return null;
+                })
+                .orElse(null);
+    }
+
+    boolean hasValue(String name) {
+        return values.containsKey(name);
+    }
+
+    Object get(String name) {
+        if (values.containsKey(name)) {
+            return values.get(name);
+        }
+        Object value = createForProperty(name);
+        if (value != null) {
+            values.put(name, value);
+        }
+        return values.get(name);
+    }
+
+    private Object[] retrieveAndResizeArray(String name, int index) {
+        Object existing = values.get(name);
+        Object[] destination;
+        if (existing != null) {
+            if (!existing.getClass().isArray()) {
+                throw new IllegalStateException("Requested get for an array, but existing value isn't an array");
+            }
+            destination = (Object[]) existing;
+            if (index >= destination.length) {
+                destination = Arrays.copyOf(destination, index + 1);
+                values.put(name, destination);
+            }
+
+        } else {
+            destination = new Object[index + 1];
+            values.put(name, destination);
+        }
+
+        return destination;
+    }
+
+    Object get(String name, int index) {
+        Object[] destination = retrieveAndResizeArray(name, index);
+
+        if (destination[index] == null) {
+            destination[index] = createForProperty(name);
+        }
+        return destination[index];
+    }
+
+    void set(String name, Object value) {
+        if (value instanceof String && properties.get(name).stream().anyMatch(t -> t.type.isEnum())) {
+            TypeTag typeTag = properties.get(name).stream()
+                    .filter(t -> t.type.isEnum())
+                    .findFirst()
+                    .orElseThrow(() ->
+                            new IllegalStateException("Expected enum type for " + name + ", but couldn't find it."));
+            Class<? extends Enum> enumClass = (Class<? extends Enum>) typeTag.type;
+            values.put(name, Enum.valueOf(enumClass, value.toString()));
+        } else {
+            values.put(name, value);
+        }
+    }
+
+    void set(String name, int index, Object value) {
+        Object[] destination = retrieveAndResizeArray(name, index);
+        destination[index] = value;
+    }
+
+    private Object getArgument(Map.Entry<String, Object> setValue) {
+        Object argument = setValue.getValue();
+        if (argument instanceof Object[]) {
+            TypeTag arrayType = properties.get(setValue.getKey()).stream()
+                    .filter(t -> t.type.isArray())
+                    .findFirst()
+                    .orElseThrow(() -> new IllegalStateException(String.format(
+                            "Received Object[] for %s but can't find corresponding type", setValue.getKey())));
+            Object[] arrayValues = (Object[]) argument;
+            Object[] destination = (Object[]) Array.newInstance(arrayType.type.getComponentType(), arrayValues.length);
+
+            for (int i = 0; i < arrayValues.length; ++i) {
+                if (arrayValues[i] instanceof BuilderDynaBean) {
+                    destination[i] = ((BuilderDynaBean) arrayValues[i]).build(Object.class);
+                } else {
+                    destination[i] = arrayValues[i];
+                }
+            }
+
+            return destination;
+        }
+        if (argument instanceof BuilderDynaBean) {
+            argument = ((BuilderDynaBean) argument).build(Object.class);
+        }
+        return argument;
+    }
+
+    Object build(Function<Object, Object>... additionalMutators) {
+        Method builderMethod;
+        try {
+            builderMethod = destinedClass.getMethod(BUILDER_METHOD_NAME);
+        } catch (NoSuchMethodException e) {
+            throw new RuntimeException(e);
+        }
+        Object source;
+        try {
+            source = builderMethod.invoke(null);
+        } catch (IllegalAccessException | InvocationTargetException e) {
+            throw new RuntimeException(e);
+        }
+        for (Map.Entry<String, Object> setValue : values.entrySet()) {
+            Object argument = getArgument(setValue);
+            Method mutator = properties.get(setValue.getKey()).stream()
+                    .filter(t -> ClassUtils.isAssignable(argument.getClass(), t.type))
+                    .findFirst()
+                    .map(a -> a.builderMethod)
+                    .orElseThrow(() -> new IllegalStateException(String.format(
+                            "Unable to find mutator for %s of type %s",
+                            setValue.getKey(), argument.getClass().getName())));
+            try {
+                source = mutator.invoke(source, argument);
+            } catch (IllegalAccessException | InvocationTargetException e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        if (additionalMutators != null) {
+            for (Function<Object, Object> mutator : additionalMutators) {
+                source = mutator.apply(source);
+            }
+        }
+
+        Method buildMethod;
+        try {
+            buildMethod = builderClass.getMethod(BUILD_METHOD_NAME);
+            return buildMethod.invoke(source);
+        } catch (IllegalAccessException | NoSuchMethodException | InvocationTargetException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    Collection<String> getPropertyNames() {
+        return properties.keySet();
+    }
+
+    List<TypeTag> getProperty(String name) {
+        if (!properties.containsKey(name)) {
+            throw new IllegalArgumentException("Unknown property: " + name);
+        }
+        return new ArrayList<>(properties.get(name));
+    }
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/DynaBeanBuilderUtils.java

@@ -0,0 +1,56 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.multilang.config;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.Arrays;
+
+public class DynaBeanBuilderUtils {
+
+    static Method getMethod(Class<?> clazz, String name, Class<?>... parameterTypes) {
+        try {
+            return clazz.getMethod(name, parameterTypes);
+        } catch (NoSuchMethodException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    static Object invokeOrFail(Method method, Object onObject, Object... arguments) {
+        try {
+            return method.invoke(onObject, arguments);
+        } catch (IllegalAccessException | InvocationTargetException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    static boolean isBuilderOrCreate(Class<?> clazz) {
+        Method buildMethod = null;
+
+        try {
+            buildMethod = clazz.getMethod("builder");
+        } catch (NoSuchMethodException e) {
+            //
+            // Ignored
+            //
+        }
+
+        boolean hasCreate = Arrays.stream(clazz.getMethods())
+                .anyMatch(m -> "create".equals(m.getName()) && m.getReturnType().isAssignableFrom(clazz));
+
+        return buildMethod != null || hasCreate;
+    }
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/DynaBeanCreateSupport.java

@@ -0,0 +1,99 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.multilang.config;
+
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.beanutils.ConvertUtilsBean;
+import org.apache.commons.lang3.StringUtils;
+
+class DynaBeanCreateSupport {
+
+    private final Class<?> destinedClass;
+    private final ConvertUtilsBean convertUtilsBean;
+    private final List<String> classPrefixSearchList;
+    private final List<TypeTag> createTypes = new ArrayList<>();
+    private Object[] createValues = null;
+
+    DynaBeanCreateSupport(
+            Class<?> destinedClass, ConvertUtilsBean convertUtilsBean, List<String> classPrefixSearchList) {
+        this.destinedClass = destinedClass;
+        this.convertUtilsBean = convertUtilsBean;
+        this.classPrefixSearchList = classPrefixSearchList;
+
+        readTypes();
+    }
+
+    private void readTypes() {
+        for (Method method : destinedClass.getMethods()) {
+            if ("create".equals(method.getName()) && method.getReturnType().isAssignableFrom(destinedClass)) {
+                createValues = new Object[method.getParameterCount()];
+                int i = 0;
+                for (Class<?> paramType : method.getParameterTypes()) {
+                    if (convertUtilsBean.lookup(paramType) != null) {
+                        createTypes.add(new TypeTag(paramType, true, null));
+                    } else {
+                        createTypes.add(new TypeTag(paramType, false, null));
+                    }
+                    ++i;
+                }
+            }
+        }
+    }
+
+    Object build() {
+
+        Method createMethod = DynaBeanBuilderUtils.getMethod(
+                destinedClass, "create", createTypes.stream().map(t -> t.type).toArray(i -> new Class<?>[i]));
+        Object arguments[] = new Object[createValues.length];
+        for (int i = 0; i < createValues.length; ++i) {
+            if (createValues[i] instanceof BuilderDynaBean) {
+                arguments[i] = ((BuilderDynaBean) createValues[i]).build(createTypes.get(i).type);
+            } else {
+                arguments[i] = createValues[i];
+            }
+        }
+        return DynaBeanBuilderUtils.invokeOrFail(createMethod, null, arguments);
+    }
+
+    public Object get(String name, int index) {
+        if (index < createValues.length) {
+            if (createTypes.get(index).hasConverter) {
+                return createValues[index];
+            } else {
+                if (createValues[index] == null) {
+                    createValues[index] = new BuilderDynaBean(
+                            createTypes.get(index).type, convertUtilsBean, null, classPrefixSearchList);
+                }
+                return createValues[index];
+            }
+        }
+        return null;
+    }
+
+    public void set(String name, int index, Object value) {
+        if (StringUtils.isEmpty(name)) {
+            if (index >= createValues.length) {
+                throw new IllegalArgumentException(String.format(
+                        "%d exceeds the maximum number of arguments (%d) for %s",
+                        index, createValues.length, destinedClass.getName()));
+            }
+            createValues[index] = value;
+        }
+    }
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/FanoutConfigBean.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.multilang.config;
+
+import lombok.Getter;
+import lombok.Setter;
+import software.amazon.awssdk.services.kinesis.KinesisAsyncClient;
+import software.amazon.kinesis.retrieval.fanout.FanOutConfig;
+
+@Getter
+@Setter
+public class FanoutConfigBean implements RetrievalConfigBuilder {
+
+    @ConfigurationSettable(configurationClass = FanOutConfig.class)
+    private int maxDescribeStreamSummaryRetries;
+
+    @ConfigurationSettable(configurationClass = FanOutConfig.class)
+    private String consumerArn;
+
+    @ConfigurationSettable(configurationClass = FanOutConfig.class)
+    private String consumerName;
+
+    @ConfigurationSettable(configurationClass = FanOutConfig.class)
+    private int maxDescribeStreamConsumerRetries;
+
+    @ConfigurationSettable(configurationClass = FanOutConfig.class)
+    private int registerStreamConsumerRetries;
+
+    @ConfigurationSettable(configurationClass = FanOutConfig.class)
+    private long retryBackoffMillis;
+
+    @Override
+    public FanOutConfig build(KinesisAsyncClient kinesisAsyncClient, MultiLangDaemonConfiguration parent) {
+        return ConfigurationSettableUtils.resolveFields(
+                this,
+                new FanOutConfig(kinesisAsyncClient)
+                        .applicationName(parent.getApplicationName())
+                        .streamName(parent.getStreamName()));
+    }
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/GracefulLeaseHandoffConfigBean.java

@@ -0,0 +1,41 @@
+/*
+ * Copyright 2024 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.multilang.config;
+
+import lombok.Getter;
+import lombok.Setter;
+import software.amazon.kinesis.leases.LeaseManagementConfig;
+
+@Getter
+@Setter
+public class GracefulLeaseHandoffConfigBean {
+
+    interface GracefulLeaseHandoffConfigBeanDelegate {
+        Long getGracefulLeaseHandoffTimeoutMillis();
+
+        void setGracefulLeaseHandoffTimeoutMillis(Long value);
+
+        Boolean getIsGracefulLeaseHandoffEnabled();
+
+        void setIsGracefulLeaseHandoffEnabled(Boolean value);
+    }
+
+    @ConfigurationSettable(configurationClass = LeaseManagementConfig.GracefulLeaseHandoffConfig.class)
+    private Long gracefulLeaseHandoffTimeoutMillis;
+
+    @ConfigurationSettable(configurationClass = LeaseManagementConfig.GracefulLeaseHandoffConfig.class)
+    private Boolean isGracefulLeaseHandoffEnabled;
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/IPropertyValueDecoder.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package software.amazon.kinesis.multilang.config;
+
+import java.util.List;
+
+/**
+ * This class captures the concept of decoding a property value to a particular Java type.
+ *
+ * @param <T>
+ */
+interface IPropertyValueDecoder<T> {
+    /**
+     * Get the value that was read from a configuration file and convert it to some type.
+     *
+     * @param propertyValue property string value that needs to be decoded.
+     * @return property value in type T
+     */
+    T decodeValue(String propertyValue);
+
+    /**
+     * Get a list of supported types for this class.
+     *
+     * @return list of supported classes.
+     */
+    List<Class<T>> getSupportedTypes();
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/IntegerPropertyValueDecoder.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package software.amazon.kinesis.multilang.config;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Get integer properties.
+ */
+class IntegerPropertyValueDecoder implements IPropertyValueDecoder<Integer> {
+
+    /**
+     * Constructor.
+     */
+    IntegerPropertyValueDecoder() {}
+
+    /**
+     * @param value property value as String
+     * @return corresponding variable in correct type
+     */
+    @Override
+    public Integer decodeValue(String value) {
+        return Integer.parseInt(value);
+    }
+
+    /**
+     * @return list of supported types
+     */
+    @Override
+    public List<Class<Integer>> getSupportedTypes() {
+        return Arrays.asList(int.class, Integer.class);
+    }
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/KinesisClientLibConfigurator.java

@@ -0,0 +1,126 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package software.amazon.kinesis.multilang.config;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.reflect.InvocationTargetException;
+import java.util.Properties;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.beanutils.BeanUtilsBean;
+import org.apache.commons.beanutils.ConvertUtilsBean;
+import org.apache.commons.lang3.Validate;
+import software.amazon.awssdk.arns.Arn;
+import software.amazon.kinesis.common.StreamIdentifier;
+
+/**
+ * KinesisClientLibConfigurator constructs a KinesisClientLibConfiguration from java properties file. The following
+ * three properties must be provided. 1) "applicationName" 2) "streamName" 3) "AwsCredentialsProvider"
+ * KinesisClientLibConfigurator will help to automatically assign the value of "workerId" if this property is not
+ * provided. In the specified properties file, any properties, which matches the variable name in
+ * KinesisClientLibConfiguration and has a corresponding "with{variableName}" setter method, will be read in, and its
+ * value in properties file will be assigned to corresponding variable in KinesisClientLibConfiguration.
+ */
+@Slf4j
+public class KinesisClientLibConfigurator {
+    private final ConvertUtilsBean convertUtilsBean;
+    private final BeanUtilsBean utilsBean;
+    private final MultiLangDaemonConfiguration configuration;
+
+    /**
+     * Constructor.
+     */
+    public KinesisClientLibConfigurator() {
+        this.convertUtilsBean = new ConvertUtilsBean();
+        this.utilsBean = new BeanUtilsBean(convertUtilsBean);
+        this.configuration = new MultiLangDaemonConfiguration(utilsBean, convertUtilsBean);
+    }
+
+    /**
+     * Return a KinesisClientLibConfiguration with variables configured as specified by the properties in config stream.
+     * Program will fail immediately, if customer provide: 1) invalid variable value. Program will log it as warning and
+     * continue, if customer provide: 1) variable with unsupported variable type. 2) a variable with name which does not
+     * match any of the variables in KinesisClientLibConfigration.
+     *
+     * @param properties a Properties object containing the configuration information
+     * @return KinesisClientLibConfiguration
+     */
+    public MultiLangDaemonConfiguration getConfiguration(Properties properties) {
+        properties.entrySet().forEach(e -> {
+            try {
+                log.info("Processing (key={}, value={})", e.getKey(), e.getValue());
+                utilsBean.setProperty(configuration, processKey((String) e.getKey()), e.getValue());
+            } catch (IllegalAccessException | InvocationTargetException ex) {
+                throw new RuntimeException(ex);
+            }
+        });
+
+        Validate.notBlank(configuration.getApplicationName(), "Application name is required");
+
+        if (configuration.getStreamArn() != null
+                && !configuration.getStreamArn().trim().isEmpty()) {
+            final Arn streamArnObj = Arn.fromString(configuration.getStreamArn());
+            StreamIdentifier.validateArn(streamArnObj);
+            // Parse out the stream Name from the Arn (and/or override existing value for Stream Name)
+            final String streamNameFromArn = streamArnObj.resource().resource();
+            configuration.setStreamName(streamNameFromArn);
+        }
+
+        Validate.notBlank(
+                configuration.getStreamName(),
+                "Stream name or Stream Arn is required. Stream Arn takes precedence if both are passed in.");
+        Validate.isTrue(
+                configuration.getKinesisCredentialsProvider().isDirty(),
+                "A basic set of AWS credentials must be provided");
+
+        return configuration;
+    }
+
+    /**
+     * @param configStream the input stream containing the configuration information
+     * @return KinesisClientLibConfiguration
+     */
+    public MultiLangDaemonConfiguration getConfiguration(InputStream configStream) {
+        Properties properties = new Properties();
+        try {
+            properties.load(configStream);
+        } catch (IOException e) {
+            String msg = "Could not load properties from the stream provided";
+            throw new IllegalStateException(msg, e);
+        } finally {
+            try {
+                configStream.close();
+            } catch (IOException e) {
+                String msg = "Encountered error while trying to close properties file.";
+                throw new IllegalStateException(msg, e);
+            }
+        }
+        return getConfiguration(properties);
+    }
+
+    /**
+     * Processes a configuration key to normalize AWS credentials provider naming. Necessary to conform to
+     * autogenerated setters.
+     * @param key the config param key
+     * @return case-configured param key name
+     */
+    String processKey(String key) {
+        if (key.toLowerCase().startsWith("awscredentialsprovider")) {
+            key = key.replaceAll("(?i)awscredentialsprovider", "awsCredentialsProvider");
+        }
+        return key;
+    }
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/MultiLangDaemonConfiguration.java

@@ -0,0 +1,531 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.multilang.config;
+
+import java.lang.reflect.InvocationTargetException;
+import java.net.URI;
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.function.Function;
+
+import lombok.Data;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.experimental.Delegate;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.beanutils.BeanUtilsBean;
+import org.apache.commons.beanutils.ConvertUtilsBean;
+import org.apache.commons.beanutils.Converter;
+import org.apache.commons.beanutils.converters.ArrayConverter;
+import org.apache.commons.beanutils.converters.StringConverter;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient;
+import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
+import software.amazon.awssdk.services.dynamodb.model.BillingMode;
+import software.amazon.awssdk.services.kinesis.KinesisAsyncClient;
+import software.amazon.awssdk.services.kinesis.KinesisAsyncClientBuilder;
+import software.amazon.kinesis.checkpoint.CheckpointConfig;
+import software.amazon.kinesis.common.ConfigsBuilder;
+import software.amazon.kinesis.common.InitialPositionInStream;
+import software.amazon.kinesis.common.InitialPositionInStreamExtended;
+import software.amazon.kinesis.common.KinesisClientUtil;
+import software.amazon.kinesis.coordinator.CoordinatorConfig;
+import software.amazon.kinesis.coordinator.Scheduler;
+import software.amazon.kinesis.leases.LeaseManagementConfig;
+import software.amazon.kinesis.leases.ShardPrioritization;
+import software.amazon.kinesis.lifecycle.LifecycleConfig;
+import software.amazon.kinesis.metrics.MetricsConfig;
+import software.amazon.kinesis.metrics.MetricsLevel;
+import software.amazon.kinesis.multilang.config.converter.DurationConverter;
+import software.amazon.kinesis.multilang.config.converter.TagConverter;
+import software.amazon.kinesis.multilang.config.converter.TagConverter.TagCollection;
+import software.amazon.kinesis.processor.ProcessorConfig;
+import software.amazon.kinesis.processor.ShardRecordProcessorFactory;
+import software.amazon.kinesis.retrieval.RetrievalConfig;
+import software.amazon.kinesis.retrieval.polling.PollingConfig;
+
+@Getter
+@Setter
+@Slf4j
+public class MultiLangDaemonConfiguration {
+
+    private static final String CREDENTIALS_DEFAULT_SEARCH_PATH = "software.amazon.awssdk.auth.credentials";
+
+    private String applicationName;
+
+    private String streamName;
+    private String streamArn;
+
+    @ConfigurationSettable(configurationClass = ConfigsBuilder.class)
+    private String tableName;
+
+    private String workerIdentifier = UUID.randomUUID().toString();
+
+    public void setWorkerId(String workerId) {
+        this.workerIdentifier = workerId;
+    }
+
+    @ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
+    private long failoverTimeMillis;
+
+    @ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
+    private Boolean enablePriorityLeaseAssignment;
+
+    @ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
+    private Boolean leaseTableDeletionProtectionEnabled;
+
+    @ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
+    private Boolean leaseTablePitrEnabled;
+
+    @ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
+    private long shardSyncIntervalMillis;
+
+    @ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
+    private boolean cleanupLeasesUponShardCompletion;
+
+    @ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
+    private boolean ignoreUnexpectedChildShards;
+
+    @ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
+    private int maxLeasesForWorker;
+
+    @ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
+    private int maxLeasesToStealAtOneTime;
+
+    @ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
+    private int initialLeaseTableReadCapacity;
+
+    @ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
+    private int initialLeaseTableWriteCapacity;
+
+    @ConfigurationSettable(configurationClass = LeaseManagementConfig.class, methodName = "initialPositionInStream")
+    @ConfigurationSettable(configurationClass = RetrievalConfig.class)
+    private InitialPositionInStreamExtended initialPositionInStreamExtended;
+
+    public InitialPositionInStream getInitialPositionInStream() {
+        if (initialPositionInStreamExtended != null) {
+            return initialPositionInStreamExtended.getInitialPositionInStream();
+        }
+        return null;
+    }
+
+    public void setInitialPositionInStream(InitialPositionInStream initialPositionInStream) {
+        this.initialPositionInStreamExtended =
+                InitialPositionInStreamExtended.newInitialPosition(initialPositionInStream);
+    }
+
+    @ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
+    private int maxLeaseRenewalThreads;
+
+    @ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
+    private long listShardsBackoffTimeInMillis;
+
+    @ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
+    private int maxListShardsRetryAttempts;
+
+    // Enables applications flush/checkpoint (if they have some data "in progress", but don't get new data for while)
+    @ConfigurationSettable(configurationClass = ProcessorConfig.class)
+    private boolean callProcessRecordsEvenForEmptyRecordList;
+
+    @ConfigurationSettable(configurationClass = CoordinatorConfig.class)
+    private long parentShardPollIntervalMillis;
+
+    @ConfigurationSettable(configurationClass = CoordinatorConfig.class)
+    private ShardPrioritization shardPrioritization;
+
+    @ConfigurationSettable(configurationClass = CoordinatorConfig.class)
+    private boolean skipShardSyncAtWorkerInitializationIfLeasesExist;
+
+    @ConfigurationSettable(configurationClass = CoordinatorConfig.class)
+    private long schedulerInitializationBackoffTimeMillis;
+
+    @ConfigurationSettable(configurationClass = CoordinatorConfig.class)
+    private CoordinatorConfig.ClientVersionConfig clientVersionConfig;
+
+    @ConfigurationSettable(configurationClass = LifecycleConfig.class)
+    private long taskBackoffTimeMillis;
+
+    @ConfigurationSettable(configurationClass = MetricsConfig.class)
+    private long metricsBufferTimeMillis;
+
+    @ConfigurationSettable(configurationClass = MetricsConfig.class)
+    private int metricsMaxQueueSize;
+
+    @ConfigurationSettable(configurationClass = MetricsConfig.class)
+    private MetricsLevel metricsLevel;
+
+    @ConfigurationSettable(configurationClass = LifecycleConfig.class, convertToOptional = true)
+    private Long logWarningForTaskAfterMillis;
+
+    @ConfigurationSettable(configurationClass = MetricsConfig.class)
+    private Set<String> metricsEnabledDimensions;
+
+    public String[] getMetricsEnabledDimensions() {
+        return metricsEnabledDimensions.toArray(new String[0]);
+    }
+
+    public void setMetricsEnabledDimensions(String[] dimensions) {
+        metricsEnabledDimensions = new HashSet<>(Arrays.asList(dimensions));
+    }
+
+    private RetrievalMode retrievalMode = RetrievalMode.DEFAULT;
+
+    private final FanoutConfigBean fanoutConfig = new FanoutConfigBean();
+
+    @Delegate(types = PollingConfigBean.PollingConfigBeanDelegate.class)
+    private final PollingConfigBean pollingConfig = new PollingConfigBean();
+
+    @Delegate(types = GracefulLeaseHandoffConfigBean.GracefulLeaseHandoffConfigBeanDelegate.class)
+    private final GracefulLeaseHandoffConfigBean gracefulLeaseHandoffConfigBean = new GracefulLeaseHandoffConfigBean();
+
+    @Delegate(
+            types = WorkerUtilizationAwareAssignmentConfigBean.WorkerUtilizationAwareAssignmentConfigBeanDelegate.class)
+    private final WorkerUtilizationAwareAssignmentConfigBean workerUtilizationAwareAssignmentConfigBean =
+            new WorkerUtilizationAwareAssignmentConfigBean();
+
+    @Delegate(types = WorkerMetricStatsTableConfigBean.WorkerMetricsTableConfigBeanDelegate.class)
+    private final WorkerMetricStatsTableConfigBean workerMetricStatsTableConfigBean =
+            new WorkerMetricStatsTableConfigBean();
+
+    @Delegate(types = CoordinatorStateTableConfigBean.CoordinatorStateConfigBeanDelegate.class)
+    private final CoordinatorStateTableConfigBean coordinatorStateTableConfigBean =
+            new CoordinatorStateTableConfigBean();
+
+    private boolean validateSequenceNumberBeforeCheckpointing;
+
+    private long shutdownGraceMillis;
+    private Integer timeoutInSeconds;
+
+    private final BuilderDynaBean kinesisCredentialsProvider;
+
+    public void setAwsCredentialsProvider(String providerString) {
+        kinesisCredentialsProvider.set("", providerString);
+    }
+
+    private final BuilderDynaBean dynamoDBCredentialsProvider;
+
+    public void setAwsCredentialsProviderDynamoDB(String providerString) {
+        dynamoDBCredentialsProvider.set("", providerString);
+    }
+
+    private final BuilderDynaBean cloudWatchCredentialsProvider;
+
+    public void setAwsCredentialsProviderCloudWatch(String providerString) {
+        cloudWatchCredentialsProvider.set("", providerString);
+    }
+
+    private final BuilderDynaBean kinesisClient;
+    private final BuilderDynaBean dynamoDbClient;
+    private final BuilderDynaBean cloudWatchClient;
+
+    private final BeanUtilsBean utilsBean;
+    private final ConvertUtilsBean convertUtilsBean;
+
+    public MultiLangDaemonConfiguration(BeanUtilsBean utilsBean, ConvertUtilsBean convertUtilsBean) {
+        this.utilsBean = utilsBean;
+        this.convertUtilsBean = convertUtilsBean;
+
+        convertUtilsBean.register(
+                new Converter() {
+                    @Override
+                    public <T> T convert(Class<T> type, Object value) {
+                        Date date = new Date(Long.parseLong(value.toString()) * 1000L);
+                        return type.cast(InitialPositionInStreamExtended.newInitialPositionAtTimestamp(date));
+                    }
+                },
+                InitialPositionInStreamExtended.class);
+
+        convertUtilsBean.register(
+                new Converter() {
+                    @Override
+                    public <T> T convert(Class<T> type, Object value) {
+                        return type.cast(MetricsLevel.valueOf(value.toString().toUpperCase()));
+                    }
+                },
+                MetricsLevel.class);
+
+        convertUtilsBean.register(
+                new Converter() {
+                    @Override
+                    public <T> T convert(Class<T> type, Object value) {
+                        return type.cast(
+                                InitialPositionInStream.valueOf(value.toString().toUpperCase()));
+                    }
+                },
+                InitialPositionInStream.class);
+
+        convertUtilsBean.register(
+                new Converter() {
+                    @Override
+                    public <T> T convert(Class<T> type, Object value) {
+                        return type.cast(CoordinatorConfig.ClientVersionConfig.valueOf(
+                                value.toString().toUpperCase()));
+                    }
+                },
+                CoordinatorConfig.ClientVersionConfig.class);
+
+        convertUtilsBean.register(
+                new Converter() {
+                    @Override
+                    public <T> T convert(Class<T> type, Object value) {
+                        return type.cast(BillingMode.valueOf(value.toString().toUpperCase()));
+                    }
+                },
+                BillingMode.class);
+
+        convertUtilsBean.register(
+                new Converter() {
+                    @Override
+                    public <T> T convert(Class<T> type, Object value) {
+                        return type.cast(URI.create(value.toString()));
+                    }
+                },
+                URI.class);
+
+        convertUtilsBean.register(
+                new Converter() {
+                    @Override
+                    public <T> T convert(Class<T> type, Object value) {
+                        return type.cast(RetrievalMode.from(value.toString()));
+                    }
+                },
+                RetrievalMode.class);
+
+        convertUtilsBean.register(
+                new Converter() {
+                    @Override
+                    public <T> T convert(final Class<T> type, final Object value) {
+                        return type.cast(Region.of(value.toString()));
+                    }
+                },
+                Region.class);
+
+        convertUtilsBean.register(new DurationConverter(), Duration.class);
+        convertUtilsBean.register(new TagConverter(), TagCollection.class);
+
+        ArrayConverter arrayConverter = new ArrayConverter(String[].class, new StringConverter());
+        arrayConverter.setDelimiter(',');
+        convertUtilsBean.register(arrayConverter, String[].class);
+        AwsCredentialsProviderPropertyValueDecoder credentialsDecoder =
+                new AwsCredentialsProviderPropertyValueDecoder();
+        Function<String, ?> converter = credentialsDecoder::decodeValue;
+
+        this.kinesisCredentialsProvider = new BuilderDynaBean(
+                AwsCredentialsProvider.class, convertUtilsBean, converter, CREDENTIALS_DEFAULT_SEARCH_PATH);
+        this.dynamoDBCredentialsProvider = new BuilderDynaBean(
+                AwsCredentialsProvider.class, convertUtilsBean, converter, CREDENTIALS_DEFAULT_SEARCH_PATH);
+        this.cloudWatchCredentialsProvider = new BuilderDynaBean(
+                AwsCredentialsProvider.class, convertUtilsBean, converter, CREDENTIALS_DEFAULT_SEARCH_PATH);
+
+        this.kinesisClient = new BuilderDynaBean(KinesisAsyncClient.class, convertUtilsBean);
+        this.dynamoDbClient = new BuilderDynaBean(DynamoDbAsyncClient.class, convertUtilsBean);
+        this.cloudWatchClient = new BuilderDynaBean(CloudWatchAsyncClient.class, convertUtilsBean);
+    }
+
+    private void setRegionForClient(String name, BuilderDynaBean client, Region region) {
+        try {
+            utilsBean.setProperty(client, "region", region);
+        } catch (IllegalAccessException | InvocationTargetException e) {
+            log.error("Failed to set region on {}", name, e);
+            throw new IllegalStateException(e);
+        }
+    }
+
+    public void setRegionName(Region region) {
+        setRegionForClient("kinesisClient", kinesisClient, region);
+        setRegionForClient("dynamoDbClient", dynamoDbClient, region);
+        setRegionForClient("cloudWatchClient", cloudWatchClient, region);
+    }
+
+    private void setEndpointForClient(String name, BuilderDynaBean client, String endpoint) {
+        try {
+            utilsBean.setProperty(client, "endpointOverride", endpoint);
+        } catch (IllegalAccessException | InvocationTargetException e) {
+            log.error("Failed to set endpoint on {}", name, e);
+            throw new IllegalStateException(e);
+        }
+    }
+
+    public void setKinesisEndpoint(String endpoint) {
+        setEndpointForClient("kinesisClient", kinesisClient, endpoint);
+    }
+
+    public void setDynamoDBEndpoint(String endpoint) {
+        setEndpointForClient("dynamoDbClient", dynamoDbClient, endpoint);
+    }
+
+    private AwsCredentialsProvider resolveCredentials(BuilderDynaBean credsBuilder) {
+        if (!credsBuilder.isDirty()) {
+            return null;
+        }
+        return credsBuilder.build(AwsCredentialsProvider.class);
+    }
+
+    private void updateCredentials(
+            BuilderDynaBean toUpdate, AwsCredentialsProvider primary, AwsCredentialsProvider secondary) {
+
+        if (toUpdate.hasValue("credentialsProvider")) {
+            return;
+        }
+
+        try {
+            if (primary != null) {
+                utilsBean.setProperty(toUpdate, "credentialsProvider", primary);
+            } else if (secondary != null) {
+                utilsBean.setProperty(toUpdate, "credentialsProvider", secondary);
+            }
+        } catch (IllegalAccessException | InvocationTargetException e) {
+            throw new RuntimeException("Unable to update credentials", e);
+        }
+    }
+
+    private void addConfigObjects(Map<Class<?>, Object> configObjects, Object... toAdd) {
+        for (Object obj : toAdd) {
+            configObjects.put(obj.getClass(), obj);
+        }
+    }
+
+    private void resolveFields(Map<Class<?>, Object> configObjects, Set<Class<?>> restrictTo, Set<Class<?>> skipIf) {
+        ConfigurationSettableUtils.resolveFields(this, configObjects, restrictTo, skipIf);
+    }
+
+    private void handleRetrievalConfig(RetrievalConfig retrievalConfig, ConfigsBuilder configsBuilder) {
+        retrievalConfig.retrievalSpecificConfig(
+                retrievalMode.builder(this).build(configsBuilder.kinesisClient(), this));
+    }
+
+    private void handleCoordinatorConfig(CoordinatorConfig coordinatorConfig) {
+        ConfigurationSettableUtils.resolveFields(
+                this.coordinatorStateTableConfigBean, coordinatorConfig.coordinatorStateTableConfig());
+    }
+
+    private void handleLeaseManagementConfig(LeaseManagementConfig leaseManagementConfig) {
+        ConfigurationSettableUtils.resolveFields(
+                this.gracefulLeaseHandoffConfigBean, leaseManagementConfig.gracefulLeaseHandoffConfig());
+        ConfigurationSettableUtils.resolveFields(
+                this.workerUtilizationAwareAssignmentConfigBean,
+                leaseManagementConfig.workerUtilizationAwareAssignmentConfig());
+        ConfigurationSettableUtils.resolveFields(
+                this.workerMetricStatsTableConfigBean,
+                leaseManagementConfig.workerUtilizationAwareAssignmentConfig().workerMetricsTableConfig());
+    }
+
+    private Object adjustKinesisHttpConfiguration(Object builderObj) {
+        if (builderObj instanceof KinesisAsyncClientBuilder) {
+            KinesisAsyncClientBuilder builder = (KinesisAsyncClientBuilder) builderObj;
+            return builder.applyMutation(KinesisClientUtil::adjustKinesisClientBuilder);
+        }
+
+        return builderObj;
+    }
+
+    @Data
+    static class ResolvedConfiguration {
+        final CoordinatorConfig coordinatorConfig;
+        final CheckpointConfig checkpointConfig;
+        final LeaseManagementConfig leaseManagementConfig;
+        final LifecycleConfig lifecycleConfig;
+        final MetricsConfig metricsConfig;
+        final ProcessorConfig processorConfig;
+        final RetrievalConfig retrievalConfig;
+
+        public Scheduler build() {
+            return new Scheduler(
+                    checkpointConfig,
+                    coordinatorConfig,
+                    leaseManagementConfig,
+                    lifecycleConfig,
+                    metricsConfig,
+                    processorConfig,
+                    retrievalConfig);
+        }
+    }
+
+    ResolvedConfiguration resolvedConfiguration(ShardRecordProcessorFactory shardRecordProcessorFactory) {
+        AwsCredentialsProvider kinesisCreds = resolveCredentials(kinesisCredentialsProvider);
+        AwsCredentialsProvider dynamoDbCreds = resolveCredentials(dynamoDBCredentialsProvider);
+        AwsCredentialsProvider cloudwatchCreds = resolveCredentials(cloudWatchCredentialsProvider);
+
+        updateCredentials(kinesisClient, kinesisCreds, kinesisCreds);
+        updateCredentials(dynamoDbClient, dynamoDbCreds, kinesisCreds);
+        updateCredentials(cloudWatchClient, cloudwatchCreds, kinesisCreds);
+
+        KinesisAsyncClient kinesisAsyncClient =
+                kinesisClient.build(KinesisAsyncClient.class, this::adjustKinesisHttpConfiguration);
+        DynamoDbAsyncClient dynamoDbAsyncClient = dynamoDbClient.build(DynamoDbAsyncClient.class);
+        CloudWatchAsyncClient cloudWatchAsyncClient = cloudWatchClient.build(CloudWatchAsyncClient.class);
+
+        ConfigsBuilder configsBuilder = new ConfigsBuilder(
+                streamName,
+                applicationName,
+                kinesisAsyncClient,
+                dynamoDbAsyncClient,
+                cloudWatchAsyncClient,
+                workerIdentifier,
+                shardRecordProcessorFactory);
+
+        Map<Class<?>, Object> configObjects = new HashMap<>();
+        addConfigObjects(configObjects, configsBuilder);
+
+        resolveFields(
+                configObjects, Collections.singleton(ConfigsBuilder.class), Collections.singleton(PollingConfig.class));
+
+        CoordinatorConfig coordinatorConfig = configsBuilder.coordinatorConfig();
+        CheckpointConfig checkpointConfig = configsBuilder.checkpointConfig();
+        LeaseManagementConfig leaseManagementConfig = configsBuilder.leaseManagementConfig();
+        LifecycleConfig lifecycleConfig = configsBuilder.lifecycleConfig();
+        MetricsConfig metricsConfig = configsBuilder.metricsConfig();
+        ProcessorConfig processorConfig = configsBuilder.processorConfig();
+        RetrievalConfig retrievalConfig = configsBuilder.retrievalConfig();
+
+        addConfigObjects(
+                configObjects,
+                coordinatorConfig,
+                checkpointConfig,
+                leaseManagementConfig,
+                lifecycleConfig,
+                metricsConfig,
+                processorConfig,
+                retrievalConfig);
+
+        handleCoordinatorConfig(coordinatorConfig);
+        handleLeaseManagementConfig(leaseManagementConfig);
+        handleRetrievalConfig(retrievalConfig, configsBuilder);
+
+        resolveFields(configObjects, null, new HashSet<>(Arrays.asList(ConfigsBuilder.class, PollingConfig.class)));
+
+        return new ResolvedConfiguration(
+                coordinatorConfig,
+                checkpointConfig,
+                leaseManagementConfig,
+                lifecycleConfig,
+                metricsConfig,
+                processorConfig,
+                retrievalConfig);
+    }
+
+    public Scheduler build(ShardRecordProcessorFactory shardRecordProcessorFactory) {
+        return resolvedConfiguration(shardRecordProcessorFactory).build();
+    }
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/PollingConfigBean.java

@@ -0,0 +1,73 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.multilang.config;
+
+import lombok.Getter;
+import lombok.Setter;
+import software.amazon.awssdk.services.kinesis.KinesisAsyncClient;
+import software.amazon.kinesis.retrieval.polling.PollingConfig;
+
+@Getter
+@Setter
+public class PollingConfigBean implements RetrievalConfigBuilder {
+
+    /**
+     * This is used to auto-generate a delegate by Lombok at {@link MultiLangDaemonConfiguration#getPollingConfig()}
+     */
+    interface PollingConfigBeanDelegate {
+
+        Integer getRetryGetRecordsInSeconds();
+
+        void setRetryGetRecordsInSeconds(Integer value);
+
+        Integer getMaxGetRecordsThreadPool();
+
+        void setMaxGetRecordsThreadPool(Integer value);
+
+        long getIdleTimeBetweenReadsInMillis();
+
+        void setIdleTimeBetweenReadsInMillis(long value);
+
+        int getMaxRecords();
+
+        void setMaxRecords(int value);
+    }
+
+    @ConfigurationSettable(configurationClass = PollingConfig.class, convertToOptional = true)
+    private Integer retryGetRecordsInSeconds;
+
+    @ConfigurationSettable(configurationClass = PollingConfig.class, convertToOptional = true)
+    private Integer maxGetRecordsThreadPool;
+
+    @ConfigurationSettable(configurationClass = PollingConfig.class)
+    private long idleTimeBetweenReadsInMillis;
+
+    @ConfigurationSettable(configurationClass = PollingConfig.class)
+    private int maxRecords;
+
+    public boolean anyPropertiesSet() {
+        return retryGetRecordsInSeconds != null
+                || maxGetRecordsThreadPool != null
+                || idleTimeBetweenReadsInMillis != 0
+                || maxRecords != 0;
+    }
+
+    @Override
+    public PollingConfig build(KinesisAsyncClient kinesisAsyncClient, MultiLangDaemonConfiguration parent) {
+        return ConfigurationSettableUtils.resolveFields(
+                this, new PollingConfig(parent.getStreamName(), kinesisAsyncClient));
+    }
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/RetrievalConfigBuilder.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.multilang.config;
+
+import software.amazon.awssdk.services.kinesis.KinesisAsyncClient;
+import software.amazon.kinesis.retrieval.RetrievalSpecificConfig;
+
+public interface RetrievalConfigBuilder {
+    /**
+     * Creates a retrieval specific configuration using the supplied parameters, and internal class parameters
+     *
+     * @param kinesisAsyncClient
+     *            the client that will be provided to the RetrievalSpecificConfig constructor
+     * @param parent
+     *            configuration parameters that this builder can access to configure it self
+     * @return a RetrievalSpecificConfig configured according to the customer's configuration.
+     */
+    public RetrievalSpecificConfig build(KinesisAsyncClient kinesisAsyncClient, MultiLangDaemonConfiguration parent);
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/RetrievalMode.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.multilang.config;
+
+import java.util.Arrays;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.Validate;
+
+@Slf4j
+public enum RetrievalMode {
+    FANOUT(MultiLangDaemonConfiguration::getFanoutConfig),
+    POLLING(MultiLangDaemonConfiguration::getPollingConfig),
+    DEFAULT(RetrievalMode::decideForDefault);
+
+    private final Function<MultiLangDaemonConfiguration, RetrievalConfigBuilder> builderFor;
+
+    public RetrievalConfigBuilder builder(MultiLangDaemonConfiguration configuration) {
+        return builderFor.apply(configuration);
+    }
+
+    RetrievalMode(Function<MultiLangDaemonConfiguration, RetrievalConfigBuilder> builderFor) {
+        this.builderFor = builderFor;
+    }
+
+    public static RetrievalMode from(String source) {
+        Validate.notEmpty(source);
+        try {
+            return RetrievalMode.valueOf(source.toUpperCase());
+        } catch (IllegalArgumentException iae) {
+            throw new IllegalArgumentException(
+                    "Unknown retrieval type '" + source + "'. Available retrieval types: " + availableRetrievalModes());
+        }
+    }
+
+    private static String availableRetrievalModes() {
+        return "(" + Arrays.stream(RetrievalMode.values()).map(Enum::name).collect(Collectors.joining(", ")) + ")";
+    }
+
+    private static RetrievalConfigBuilder decideForDefault(MultiLangDaemonConfiguration configuration) {
+        if (configuration.getPollingConfig().anyPropertiesSet()) {
+            log.warn("Some polling properties have been set, defaulting to polling. "
+                    + "To switch to Fanout either add `RetrievalMode=FANOUT` to your "
+                    + "properties or remove the any configuration for polling.");
+            return configuration.getPollingConfig();
+        }
+        return configuration.getFanoutConfig();
+    }
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/TypeTag.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.multilang.config;
+
+import java.lang.reflect.Method;
+
+import lombok.Data;
+
+@Data
+class TypeTag {
+    final Class<?> type;
+    final boolean hasConverter;
+    final Method builderMethod;
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/WorkerMetricStatsTableConfigBean.java

@@ -0,0 +1,82 @@
+/*
+ * Copyright 2024 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.multilang.config;
+
+import lombok.Getter;
+import lombok.Setter;
+import software.amazon.awssdk.services.dynamodb.model.BillingMode;
+import software.amazon.kinesis.leases.LeaseManagementConfig.WorkerMetricsTableConfig;
+import software.amazon.kinesis.multilang.config.converter.TagConverter.TagCollection;
+
+@Getter
+@Setter
+public class WorkerMetricStatsTableConfigBean {
+
+    interface WorkerMetricsTableConfigBeanDelegate {
+        String getWorkerMetricsTableName();
+
+        void setWorkerMetricsTableName(String value);
+
+        BillingMode getWorkerMetricsBillingMode();
+
+        void setWorkerMetricsBillingMode(BillingMode value);
+
+        long getWorkerMetricsReadCapacity();
+
+        void setWorkerMetricsReadCapacity(long value);
+
+        long getWorkerMetricsWriteCapacity();
+
+        void setWorkerMetricsWriteCapacity(long value);
+
+        Boolean getWorkerMetricsPointInTimeRecoveryEnabled();
+
+        void setWorkerMetricsPointInTimeRecoveryEnabled(Boolean value);
+
+        Boolean getWorkerMetricsDeletionProtectionEnabled();
+
+        void setWorkerMetricsDeletionProtectionEnabled(Boolean value);
+
+        TagCollection getWorkerMetricsTags();
+
+        void setWorkerMetricsTags(TagCollection value);
+    }
+
+    @ConfigurationSettable(configurationClass = WorkerMetricsTableConfig.class, methodName = "tableName")
+    private String workerMetricsTableName;
+
+    @ConfigurationSettable(configurationClass = WorkerMetricsTableConfig.class, methodName = "billingMode")
+    private BillingMode workerMetricsBillingMode;
+
+    @ConfigurationSettable(configurationClass = WorkerMetricsTableConfig.class, methodName = "readCapacity")
+    private long workerMetricsReadCapacity;
+
+    @ConfigurationSettable(configurationClass = WorkerMetricsTableConfig.class, methodName = "writeCapacity")
+    private long workerMetricsWriteCapacity;
+
+    @ConfigurationSettable(
+            configurationClass = WorkerMetricsTableConfig.class,
+            methodName = "pointInTimeRecoveryEnabled")
+    private Boolean workerMetricsPointInTimeRecoveryEnabled;
+
+    @ConfigurationSettable(
+            configurationClass = WorkerMetricsTableConfig.class,
+            methodName = "deletionProtectionEnabled")
+    private Boolean workerMetricsDeletionProtectionEnabled;
+
+    @ConfigurationSettable(configurationClass = WorkerMetricsTableConfig.class, methodName = "tags")
+    private TagCollection workerMetricsTags;
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/WorkerUtilizationAwareAssignmentConfigBean.java

@@ -0,0 +1,106 @@
+/*
+ * Copyright 2024 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.multilang.config;
+
+import java.time.Duration;
+
+import lombok.Getter;
+import lombok.Setter;
+import software.amazon.kinesis.leases.LeaseManagementConfig.WorkerUtilizationAwareAssignmentConfig;
+
+@Getter
+@Setter
+public class WorkerUtilizationAwareAssignmentConfigBean {
+
+    interface WorkerUtilizationAwareAssignmentConfigBeanDelegate {
+        long getInMemoryWorkerMetricsCaptureFrequencyMillis();
+
+        void setInMemoryWorkerMetricsCaptureFrequencyMillis(long value);
+
+        long getWorkerMetricsReporterFreqInMillis();
+
+        void setWorkerMetricsReporterFreqInMillis(long value);
+
+        int getNoOfPersistedMetricsPerWorkerMetrics();
+
+        void setNoOfPersistedMetricsPerWorkerMetrics(int value);
+
+        Boolean getDisableWorkerMetrics();
+
+        void setDisableWorkerMetrics(Boolean value);
+
+        double getMaxThroughputPerHostKBps();
+
+        void setMaxThroughputPerHostKBps(double value);
+
+        int getDampeningPercentage();
+
+        void setDampeningPercentage(int value);
+
+        int getReBalanceThresholdPercentage();
+
+        void setReBalanceThresholdPercentage(int value);
+
+        Boolean getAllowThroughputOvershoot();
+
+        void setAllowThroughputOvershoot(Boolean value);
+
+        int getVarianceBalancingFrequency();
+
+        void setVarianceBalancingFrequency(int value);
+
+        double getWorkerMetricsEMAAlpha();
+
+        void setWorkerMetricsEMAAlpha(double value);
+
+        void setStaleWorkerMetricsEntryCleanupDuration(Duration value);
+
+        Duration getStaleWorkerMetricsEntryCleanupDuration();
+    }
+
+    @ConfigurationSettable(configurationClass = WorkerUtilizationAwareAssignmentConfig.class)
+    private long inMemoryWorkerMetricsCaptureFrequencyMillis;
+
+    @ConfigurationSettable(configurationClass = WorkerUtilizationAwareAssignmentConfig.class)
+    private long workerMetricsReporterFreqInMillis;
+
+    @ConfigurationSettable(configurationClass = WorkerUtilizationAwareAssignmentConfig.class)
+    private int noOfPersistedMetricsPerWorkerMetrics;
+
+    @ConfigurationSettable(configurationClass = WorkerUtilizationAwareAssignmentConfig.class)
+    private Boolean disableWorkerMetrics;
+
+    @ConfigurationSettable(configurationClass = WorkerUtilizationAwareAssignmentConfig.class)
+    private double maxThroughputPerHostKBps;
+
+    @ConfigurationSettable(configurationClass = WorkerUtilizationAwareAssignmentConfig.class)
+    private int dampeningPercentage;
+
+    @ConfigurationSettable(configurationClass = WorkerUtilizationAwareAssignmentConfig.class)
+    private int reBalanceThresholdPercentage;
+
+    @ConfigurationSettable(configurationClass = WorkerUtilizationAwareAssignmentConfig.class)
+    private Boolean allowThroughputOvershoot;
+
+    @ConfigurationSettable(configurationClass = WorkerUtilizationAwareAssignmentConfig.class)
+    private int varianceBalancingFrequency;
+
+    @ConfigurationSettable(configurationClass = WorkerUtilizationAwareAssignmentConfig.class)
+    private double workerMetricsEMAAlpha;
+
+    @ConfigurationSettable(configurationClass = WorkerUtilizationAwareAssignmentConfig.class)
+    private Duration staleWorkerMetricsEntryCleanupDuration;
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/converter/DurationConverter.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright 2024 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.multilang.config.converter;
+
+import java.time.Duration;
+
+import org.apache.commons.beanutils.Converter;
+
+/**
+ * Converter that converts Duration text representation to a Duration object.
+ * Refer to {@code Duration.parse} javadocs for the exact text representation.
+ */
+public class DurationConverter implements Converter {
+
+    @Override
+    public <T> T convert(Class<T> type, Object value) {
+        if (value == null) {
+            return null;
+        }
+
+        if (type != Duration.class) {
+            throw new ConversionException("Can only convert to Duration");
+        }
+
+        String durationString = value.toString().trim();
+        final Duration duration = Duration.parse(durationString);
+        if (duration.isNegative()) {
+            throw new ConversionException("Negative values are not permitted for duration: " + durationString);
+        }
+
+        return type.cast(duration);
+    }
+
+    public static class ConversionException extends RuntimeException {
+        public ConversionException(String message) {
+            super(message);
+        }
+    }
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/converter/TagConverter.java

@@ -0,0 +1,67 @@
+/*
+ * Copyright 2024 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.multilang.config.converter;
+
+import java.util.ArrayList;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.beanutils.Converter;
+import software.amazon.awssdk.services.dynamodb.model.Tag;
+
+/**
+ * Converter that converts to a Collection of Tag object.
+ * The text format accepted are as follows:
+ * tagPropertyName = key1=value1,key2=value2,...
+ */
+@Slf4j
+public class TagConverter implements Converter {
+
+    @Override
+    public <T> T convert(Class<T> type, Object value) {
+        if (value == null) {
+            return null;
+        }
+
+        if (!type.isAssignableFrom(TagCollection.class)) {
+            throw new ConversionException("Can only convert to Collection<Tag>");
+        }
+
+        final TagCollection collection = new TagCollection();
+        final String tagString = value.toString().trim();
+        final String[] keyValuePairs = tagString.split(",");
+        for (String keyValuePair : keyValuePairs) {
+            final String[] tokens = keyValuePair.trim().split("=");
+            if (tokens.length != 2) {
+                log.warn("Invalid tag {}, ignoring it", keyValuePair);
+                continue;
+            }
+            final Tag tag =
+                    Tag.builder().key(tokens[0].trim()).value(tokens[1].trim()).build();
+            log.info("Created tag {}", tag);
+            collection.add(tag);
+        }
+
+        return type.cast(collection);
+    }
+
+    public static class ConversionException extends RuntimeException {
+        public ConversionException(String message) {
+            super(message);
+        }
+    }
+
+    public static class TagCollection extends ArrayList<Tag> {}
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/CheckpointMessage.java

@@ -1,20 +1,21 @@
 /*
- *  Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- *  Licensed under the Amazon Software License (the "License").
- *  You may not use this file except in compliance with the License.
- *  A copy of the License is located at
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
- *  http://aws.amazon.com/asl/
- *
- *  or in the "license" file accompanying this file. This file is distributed
- *  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- *  express or implied. See the License for the specific language governing
- *  permissions and limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
-package com.amazonaws.services.kinesis.multilang.messages;
+package software.amazon.kinesis.multilang.messages;
 
 import lombok.Getter;
+import lombok.NoArgsConstructor;
 import lombok.Setter;
 
 /**
@@ -22,6 +23,7 @@ import lombok.Setter;
  * checkpoint. The processor sends back a checkpoint message as an acknowledgement that it attempted to checkpoint along
  * with an error message which corresponds to the names of exceptions that a checkpointer can throw.
  */
+@NoArgsConstructor
 @Getter
 @Setter
 public class CheckpointMessage extends Message {
@@ -34,6 +36,7 @@ public class CheckpointMessage extends Message {
      * The checkpoint this message is about.
      */
     private String sequenceNumber;
+
     private Long subSequenceNumber;
 
     /**
@@ -41,15 +44,9 @@ public class CheckpointMessage extends Message {
      */
     private String error;
 
-    /**
-     * Default constructor.
-     */
-    public CheckpointMessage() {
-    }
-
     /**
      * Convenience constructor.
-     * 
+     *
      * @param sequenceNumber
      *            The sequence number that this message is about.
      * @param subSequenceNumber
@@ -65,5 +62,4 @@ public class CheckpointMessage extends Message {
             this.setError(throwable.getClass().getSimpleName());
         }
     }
-
 }

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/InitializeMessage.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package software.amazon.kinesis.multilang.messages;
+
+import lombok.Getter;
+import lombok.Setter;
+import software.amazon.kinesis.lifecycle.events.InitializationInput;
+
+/**
+ * An initialize message is sent to the client's subprocess to indicate that it should perform its initialization steps.
+ */
+@Getter
+@Setter
+public class InitializeMessage extends Message {
+    /**
+     * The name used for the action field in {@link Message}.
+     */
+    public static final String ACTION = "initialize";
+
+    /**
+     * The shard id that this processor is getting initialized for.
+     */
+    private String shardId;
+
+    private String sequenceNumber;
+    private Long subSequenceNumber;
+
+    /**
+     * Default constructor.
+     */
+    public InitializeMessage() {}
+
+    /**
+     * Convenience constructor.
+     *
+     * @param initializationInput {@link InitializationInput}
+     */
+    public InitializeMessage(InitializationInput initializationInput) {
+        this.shardId = initializationInput.shardId();
+        if (initializationInput.extendedSequenceNumber() != null) {
+            this.sequenceNumber = initializationInput.extendedSequenceNumber().sequenceNumber();
+            this.subSequenceNumber =
+                    initializationInput.extendedSequenceNumber().subSequenceNumber();
+        } else {
+            this.sequenceNumber = null;
+            this.subSequenceNumber = null;
+        }
+    }
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/JsonFriendlyRecord.java

@@ -0,0 +1,67 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package software.amazon.kinesis.multilang.messages;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.AllArgsConstructor;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.NonNull;
+import lombok.Setter;
+import lombok.ToString;
+import software.amazon.kinesis.retrieval.KinesisClientRecord;
+
+/**
+ * Class for encoding Record objects to json. Needed because Records have byte buffers for their data field which causes
+ * problems for the json library we're using.
+ */
+@NoArgsConstructor
+@AllArgsConstructor
+@Getter
+@Setter
+@EqualsAndHashCode
+@ToString
+public class JsonFriendlyRecord {
+    private byte[] data;
+    private String partitionKey;
+    private String sequenceNumber;
+    private Long approximateArrivalTimestamp;
+    private Long subSequenceNumber;
+
+    public static String ACTION = "record";
+
+    public static JsonFriendlyRecord fromKinesisClientRecord(@NonNull final KinesisClientRecord record) {
+        byte[] data;
+        if (record.data() == null) {
+            data = null;
+        } else if (record.data().hasArray()) {
+            data = record.data().array();
+        } else {
+            data = new byte[record.data().limit()];
+            record.data().get(data);
+        }
+        Long approximateArrival = record.approximateArrivalTimestamp() == null
+                ? null
+                : record.approximateArrivalTimestamp().toEpochMilli();
+        return new JsonFriendlyRecord(
+                data, record.partitionKey(), record.sequenceNumber(), approximateArrival, record.subSequenceNumber());
+    }
+
+    @JsonProperty
+    public String getAction() {
+        return ACTION;
+    }
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/LeaseLostMessage.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.multilang.messages;
+
+/**
+ * Used to indicate to the client that the record process has lost its lease.
+ */
+public class LeaseLostMessage extends Message {
+
+    public static final String ACTION = "leaseLost";
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/Message.java

@@ -0,0 +1,66 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package software.amazon.kinesis.multilang.messages;
+
+import com.fasterxml.jackson.annotation.JsonSubTypes;
+import com.fasterxml.jackson.annotation.JsonSubTypes.Type;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+/**
+ * Abstract class for all messages that are sent to the client's process.
+ */
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "action")
+@JsonSubTypes({
+    @Type(value = CheckpointMessage.class, name = CheckpointMessage.ACTION),
+    @Type(value = InitializeMessage.class, name = InitializeMessage.ACTION),
+    @Type(value = ProcessRecordsMessage.class, name = ProcessRecordsMessage.ACTION),
+    @Type(value = ShutdownMessage.class, name = ShutdownMessage.ACTION),
+    @Type(value = StatusMessage.class, name = StatusMessage.ACTION),
+    @Type(value = ShutdownRequestedMessage.class, name = ShutdownRequestedMessage.ACTION),
+    @Type(value = LeaseLostMessage.class, name = LeaseLostMessage.ACTION),
+    @Type(value = ShardEndedMessage.class, name = ShardEndedMessage.ACTION),
+})
+public abstract class Message {
+
+    private ObjectMapper mapper = new ObjectMapper();
+
+    /**
+     * Default constructor.
+     */
+    public Message() {}
+
+    /**
+     *
+     * @param objectMapper An object mapper.
+     * @return this
+     */
+    Message withObjectMapper(ObjectMapper objectMapper) {
+        this.mapper = objectMapper;
+        return this;
+    }
+
+    /**
+     *
+     * @return A JSON representation of this object.
+     */
+    public String toString() {
+        try {
+            return mapper.writeValueAsString(this);
+        } catch (Exception e) {
+            return super.toString();
+        }
+    }
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/ProcessRecordsMessage.java

@@ -0,0 +1,62 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package software.amazon.kinesis.multilang.messages;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import lombok.Getter;
+import lombok.Setter;
+import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput;
+import software.amazon.kinesis.retrieval.KinesisClientRecord;
+
+/**
+ * A message to indicate to the client's process that it should process a list of records.
+ */
+@Getter
+@Setter
+public class ProcessRecordsMessage extends Message {
+    /**
+     * The name used for the action field in {@link Message}.
+     */
+    public static final String ACTION = "processRecords";
+
+    /**
+     * The records that the client's process needs to handle.
+     */
+    private List<JsonFriendlyRecord> records;
+
+    private Long millisBehindLatest;
+
+    /**
+     * Default constructor.
+     */
+    public ProcessRecordsMessage() {}
+
+    /**
+     * Convenience constructor.
+     *
+     * @param processRecordsInput
+     *            the process records input to be sent to the child
+     */
+    public ProcessRecordsMessage(ProcessRecordsInput processRecordsInput) {
+        this.millisBehindLatest = processRecordsInput.millisBehindLatest();
+        List<JsonFriendlyRecord> recordMessages = new ArrayList<>();
+        for (KinesisClientRecord record : processRecordsInput.records()) {
+            recordMessages.add(JsonFriendlyRecord.fromKinesisClientRecord(record));
+        }
+        this.setRecords(recordMessages);
+    }
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/ShardEndedMessage.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.multilang.messages;
+
+/**
+ * Used to indicate to the client that the shard has ended.
+ */
+public class ShardEndedMessage extends Message {
+    public static final String ACTION = "shardEnded";
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/ShutdownMessage.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package software.amazon.kinesis.multilang.messages;
+
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import software.amazon.kinesis.lifecycle.ShutdownReason;
+
+/**
+ * A message to indicate to the client's process that it should shutdown and then terminate.
+ */
+@NoArgsConstructor
+@Getter
+@Setter
+public class ShutdownMessage extends Message {
+    /**
+     * The name used for the action field in {@link Message}.
+     */
+    public static final String ACTION = "shutdown";
+
+    /**
+     * The reason for shutdown, e.g. SHARD_END or LEASE_LOST
+     */
+    private String reason;
+
+    public ShutdownMessage(final ShutdownReason reason) {
+        if (reason != null) {
+            this.reason = String.valueOf(reason);
+        }
+    }
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/ShutdownRequestedMessage.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package software.amazon.kinesis.multilang.messages;
+
+import lombok.NoArgsConstructor;
+
+/**
+ * A message to indicate to the client's process that shutdown is requested.
+ */
+@NoArgsConstructor
+public class ShutdownRequestedMessage extends Message {
+    /**
+     * The name used for the action field in {@link Message}.
+     */
+    public static final String ACTION = "shutdownRequested";
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/StatusMessage.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package software.amazon.kinesis.multilang.messages;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+/**
+ * A message sent by the client's process to indicate to the record processor that it completed a particular action.
+ */
+@NoArgsConstructor
+@AllArgsConstructor
+@Getter
+@Setter
+public class StatusMessage extends Message {
+    /**
+     * The name used for the action field in {@link Message}.
+     */
+    public static final String ACTION = "status";
+
+    /**
+     * The name of the most recently received action.
+     */
+    private String responseFor;
+}

amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/package-info.java

@@ -1,20 +1,20 @@
 /*
- * Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 /**
  * This package provides a KCL application which implements the multi language protocol. The multi language protocol
- * defines a system for communication  between a KCL multi-lang application and another process (referred to as the
+ * defines a system for communication between a KCL multi-lang application and another process (referred to as the
  * "child process") over STDIN and STDOUT of the child process. The units of communication are JSON messages which
  * represent the actions the receiving entity should perform. The child process is responsible for reacting
  * appropriately to four different messages: initialize, processRecords, checkpoint, and shutdown. The KCL multi-lang
@@ -39,8 +39,15 @@
  *   "error"      : "<NameOfException>"
  * }
  *
- * { "action" : "shutdown",
- *   "reason" : "<TERMINATE|ZOMBIE>"
+ * { "action"     : "leaseLost",
+ * }
+ *
+ * { "action"     : "shardEnded",
+ *   "checkpoint" : "<SHARD_END>",
+ * }
+ *
+ * { "action"     : "shutdownRequested",
+ *   "checkpoint" : "<sequence number>"
  * }
  * </pre>
  *
@@ -93,13 +100,29 @@
  * <li>Begin reading line from STDIN to receive next action</li>
  * </ol>
  *
- * <h4>Shutdown</h4>
+ * <h4>LeaseLost</h4>
  *
  * <ol>
- * <li>Read a "shutdown" action from STDIN</li>
- * <li>Perform shutdown tasks (you may write a checkpoint message at any time)</li>
+ * <li>Read a "leaseLost" action from STDIN</li>
+ * <li>Perform lease lost tasks (you will not be able to checkpoint at this time, since the worker doesn't hold the
+ * lease)</li>
+ * <li>Write "status" message to STDOUT to indicate you are done.</li>
+ * </ol>
+ *
+ * <h4>ShardEnded</h4>
+ *
+ * <ol>
+ * <li>Read a "shardEnded" action from STDIN</li>
+ * <li>Perform shutdown tasks (you should write a checkpoint message at any time)</li>
+ * <li>Write "status" message to STDOUT to indicate you are done.</li>
+ * </ol>
+ *
+ * <h4>ShutdownRequested</h4>
+ *
+ * <ol>
+ * <li>Read a "shutdownRequested" action from STDIN</li>
+ * <li>Perform shutdown requested related tasks (you may write a checkpoint message at any time)</li>
  * <li>Write "status" message to STDOUT to indicate you are done.</li>
- * <li>Begin reading line from STDIN to receive next action</li>
  * </ol>
  *
  * <h4>Checkpoint</h4>
@@ -119,7 +142,6 @@
  * Jackson doc for more details)</a> MIME is the basis of most base64 encoding variants including <a
  * href="http://tools.ietf.org/html/rfc3548.html">RFC 3548</a> which is the standard used by Python's <a
  * href="https://docs.python.org/2/library/base64.html">base64</a> module.
- * 
+ *
  */
-package com.amazonaws.services.kinesis.multilang;
-
+package software.amazon.kinesis.multilang;

amazon-kinesis-client-multilang/src/main/resources/logback.xml

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+  -->
+<configuration>
+    <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <pattern>%d [%thread] %-5level %logger{36} [%mdc{ShardId:-NONE}] - %msg %n</pattern>
+        </encoder>
+    </appender>
+
+    <root level="INFO">
+        <appender-ref ref="CONSOLE" />
+    </root>
+</configuration>

amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/Matchers.java

@@ -1,28 +1,27 @@
 /*
- *  Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- *  Licensed under the Amazon Software License (the "License").
- *  You may not use this file except in compliance with the License.
- *  A copy of the License is located at
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
- *  http://aws.amazon.com/asl/
- *
- *  or in the "license" file accompanying this file. This file is distributed
- *  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- *  express or implied. See the License for the specific language governing
- *  permissions and limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
-package com.amazonaws.services.kinesis.multilang;
-
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.nullValue;
+package software.amazon.kinesis.multilang;
 
 import org.hamcrest.Description;
 import org.hamcrest.Matcher;
 import org.hamcrest.TypeSafeDiagnosingMatcher;
+import software.amazon.kinesis.lifecycle.events.InitializationInput;
+import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber;
 
-import com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber;
-import com.amazonaws.services.kinesis.clientlibrary.types.InitializationInput;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.nullValue;
 
 public class Matchers {
 
@@ -36,19 +35,19 @@ public class Matchers {
         private final Matcher<ExtendedSequenceNumber> sequenceNumberMatcher;
 
         public InitializationInputMatcher(InitializationInput input) {
-            shardIdMatcher = equalTo(input.getShardId());
-            sequenceNumberMatcher = withSequence(input.getExtendedSequenceNumber());
+            shardIdMatcher = equalTo(input.shardId());
+            sequenceNumberMatcher = withSequence(input.extendedSequenceNumber());
         }
 
         @Override
         protected boolean matchesSafely(final InitializationInput item, Description mismatchDescription) {
 
             boolean matches = true;
-            if (!shardIdMatcher.matches(item.getShardId())) {
+            if (!shardIdMatcher.matches(item.shardId())) {
                 matches = false;
-                shardIdMatcher.describeMismatch(item.getShardId(), mismatchDescription);
+                shardIdMatcher.describeMismatch(item.shardId(), mismatchDescription);
             }
-            if (!sequenceNumberMatcher.matches(item.getExtendedSequenceNumber())) {
+            if (!sequenceNumberMatcher.matches(item.extendedSequenceNumber())) {
                 matches = false;
                 sequenceNumberMatcher.describeMismatch(item, mismatchDescription);
             }
@@ -58,8 +57,12 @@ public class Matchers {
 
         @Override
         public void describeTo(Description description) {
-            description.appendText("An InitializationInput matching: { shardId: ").appendDescriptionOf(shardIdMatcher)
-                    .appendText(", sequenceNumber: ").appendDescriptionOf(sequenceNumberMatcher).appendText(" }");
+            description
+                    .appendText("An InitializationInput matching: { shardId: ")
+                    .appendDescriptionOf(shardIdMatcher)
+                    .appendText(", sequenceNumber: ")
+                    .appendDescriptionOf(sequenceNumberMatcher)
+                    .appendText(" }");
         }
     }
 
@@ -76,19 +79,19 @@ public class Matchers {
         private final Matcher<Long> subSequenceNumberMatcher;
 
         public ExtendedSequenceNumberMatcher(ExtendedSequenceNumber extendedSequenceNumber) {
-            sequenceNumberMatcher = equalTo(extendedSequenceNumber.getSequenceNumber());
-            subSequenceNumberMatcher = equalTo(extendedSequenceNumber.getSubSequenceNumber());
+            sequenceNumberMatcher = equalTo(extendedSequenceNumber.sequenceNumber());
+            subSequenceNumberMatcher = equalTo(extendedSequenceNumber.subSequenceNumber());
         }
 
         @Override
         protected boolean matchesSafely(ExtendedSequenceNumber item, Description mismatchDescription) {
 
             boolean matches = true;
-            if (!sequenceNumberMatcher.matches(item.getSequenceNumber())) {
+            if (!sequenceNumberMatcher.matches(item.sequenceNumber())) {
                 matches = false;
                 mismatchDescription.appendDescriptionOf(sequenceNumberMatcher);
             }
-            if (!subSequenceNumberMatcher.matches(item.getSubSequenceNumber())) {
+            if (!subSequenceNumberMatcher.matches(item.subSequenceNumber())) {
                 matches = false;
                 mismatchDescription.appendDescriptionOf(subSequenceNumberMatcher);
             }
@@ -98,10 +101,11 @@ public class Matchers {
 
         @Override
         public void describeTo(Description description) {
-            description.appendText("An ExtendedSequenceNumber matching: { sequenceNumber: ")
-                    .appendDescriptionOf(sequenceNumberMatcher).appendText(", subSequenceNumber: ")
+            description
+                    .appendText("An ExtendedSequenceNumber matching: { sequenceNumber: ")
+                    .appendDescriptionOf(sequenceNumberMatcher)
+                    .appendText(", subSequenceNumber: ")
                     .appendDescriptionOf(subSequenceNumberMatcher);
         }
     }
-
 }

amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/MessageReaderTest.java

@@ -1,50 +1,41 @@
 /*
- * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
-package com.amazonaws.services.kinesis.multilang;
+package software.amazon.kinesis.multilang;
 
 import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
-
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import org.junit.Assert;
-
-import org.junit.Before;
 import org.junit.Test;
 import org.mockito.Mockito;
 import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
-
-import com.amazonaws.services.kinesis.multilang.messages.Message;
-import com.amazonaws.services.kinesis.multilang.messages.StatusMessage;
-import com.fasterxml.jackson.databind.ObjectMapper;
+import software.amazon.kinesis.multilang.messages.Message;
+import software.amazon.kinesis.multilang.messages.StatusMessage;
 
 public class MessageReaderTest {
 
-    private static final String shardId = "shard-123";
+    private static final String SHARD_ID = "shard-123";
 
-    @Before
-    public void setup() {
-
-    }
-
-    /*
+    /**
      * This line is based on the definition of the protocol for communication between the KCL record processor and
      * the client's process.
      */
@@ -52,7 +43,7 @@ public class MessageReaderTest {
         return String.format("{\"action\":\"checkpoint\", \"checkpoint\":\"%s\"}", sequenceNumber);
     }
 
-    /*
+    /**
      * This line is based on the definition of the protocol for communication between the KCL record processor and
      * the client's process.
      */
@@ -83,18 +74,19 @@ public class MessageReaderTest {
 
     @Test
     public void runLoopGoodInputTest() {
-        String[] sequenceNumbers = new String[] { "123", "456", "789" };
-        String[] responseFors = new String[] { "initialize", "processRecords", "processRecords", "shutdown" };
+        String[] sequenceNumbers = new String[] {"123", "456", "789"};
+        String[] responseFors = new String[] {"initialize", "processRecords", "processRecords", "shutdown"};
         InputStream stream = buildInputStreamOfGoodInput(sequenceNumbers, responseFors);
         MessageReader reader =
-                new MessageReader().initialize(stream, shardId, new ObjectMapper(), Executors.newCachedThreadPool());
+                new MessageReader().initialize(stream, SHARD_ID, new ObjectMapper(), Executors.newCachedThreadPool());
 
         for (String responseFor : responseFors) {
-            StatusMessage statusMessage = null;
             try {
                 Message message = reader.getNextMessageFromSTDOUT().get();
                 if (message instanceof StatusMessage) {
-                    Assert.assertEquals("The status message's responseFor field should have been correct", responseFor,
+                    Assert.assertEquals(
+                            "The status message's responseFor field should have been correct",
+                            responseFor,
                             ((StatusMessage) message).getResponseFor());
                 }
             } catch (InterruptedException | ExecutionException e) {
@@ -105,19 +97,19 @@ public class MessageReaderTest {
 
     @Test
     public void drainInputTest() throws InterruptedException, ExecutionException {
-        String[] sequenceNumbers = new String[] { "123", "456", "789" };
-        String[] responseFors = new String[] { "initialize", "processRecords", "processRecords", "shutdown" };
+        String[] sequenceNumbers = new String[] {"123", "456", "789"};
+        String[] responseFors = new String[] {"initialize", "processRecords", "processRecords", "shutdown"};
         InputStream stream = buildInputStreamOfGoodInput(sequenceNumbers, responseFors);
 
         MessageReader reader =
-                new MessageReader().initialize(stream, shardId, new ObjectMapper(), Executors.newCachedThreadPool());
+                new MessageReader().initialize(stream, SHARD_ID, new ObjectMapper(), Executors.newCachedThreadPool());
         Future<Boolean> drainFuture = reader.drainSTDOUT();
         Boolean drainResult = drainFuture.get();
         Assert.assertNotNull(drainResult);
         Assert.assertTrue(drainResult);
     }
 
-    /*
+    /**
      * readValue should fail safely and just continue looping
      */
     @Test
@@ -125,25 +117,26 @@ public class MessageReaderTest {
         BufferedReader bufferReader = Mockito.mock(BufferedReader.class);
         try {
             Mockito.doAnswer(new Answer() {
-                private boolean returnedOnce = false;
+                        private boolean returnedOnce = false;
 
-                @Override
-                public Object answer(InvocationOnMock invocation) throws Throwable {
-                    if (returnedOnce) {
-                        return "{\"action\":\"status\",\"responseFor\":\"processRecords\"}";
-                    } else {
-                        returnedOnce = true;
-                        return "{\"action\":\"shutdown\",\"reason\":\"ZOMBIE\"}";
-                    }
-                }
-            }).when(bufferReader).readLine();
+                        @Override
+                        public Object answer(InvocationOnMock invocation) throws Throwable {
+                            if (returnedOnce) {
+                                return "{\"action\":\"status\",\"responseFor\":\"processRecords\"}";
+                            } else {
+                                returnedOnce = true;
+                                return "{\"action\":\"shutdown\",\"reason\":\"ZOMBIE\"}";
+                            }
+                        }
+                    })
+                    .when(bufferReader)
+                    .readLine();
         } catch (IOException e) {
             Assert.fail("There shouldn't be an exception while setting up this mock.");
         }
 
-        MessageReader reader =
-                new MessageReader().initialize(bufferReader, shardId, new ObjectMapper(),
-                        Executors.newCachedThreadPool());
+        MessageReader reader = new MessageReader()
+                .initialize(bufferReader, SHARD_ID, new ObjectMapper(), Executors.newCachedThreadPool());
 
         try {
             reader.getNextMessageFromSTDOUT().get();
@@ -157,7 +150,7 @@ public class MessageReaderTest {
     public void messageReaderBuilderTest() {
         InputStream stream = new ByteArrayInputStream("".getBytes());
         MessageReader reader =
-                new MessageReader().initialize(stream, shardId, new ObjectMapper(), Executors.newCachedThreadPool());
+                new MessageReader().initialize(stream, SHARD_ID, new ObjectMapper(), Executors.newCachedThreadPool());
         Assert.assertNotNull(reader);
     }
 
@@ -166,7 +159,7 @@ public class MessageReaderTest {
         BufferedReader input = Mockito.mock(BufferedReader.class);
         Mockito.doThrow(IOException.class).when(input).readLine();
         MessageReader reader =
-                new MessageReader().initialize(input, shardId, new ObjectMapper(), Executors.newCachedThreadPool());
+                new MessageReader().initialize(input, SHARD_ID, new ObjectMapper(), Executors.newCachedThreadPool());
 
         Future<Message> readTask = reader.getNextMessageFromSTDOUT();
 
@@ -174,7 +167,8 @@ public class MessageReaderTest {
             readTask.get();
             Assert.fail("The reading task should have failed due to an IOException.");
         } catch (InterruptedException e) {
-            Assert.fail("The reading task should not have been interrupted. It should have failed due to an IOException.");
+            Assert.fail(
+                    "The reading task should not have been interrupted. It should have failed due to an IOException.");
         } catch (ExecutionException e) {
             // Yay!!
         }
@@ -184,7 +178,7 @@ public class MessageReaderTest {
     public void noMoreMessagesTest() throws InterruptedException {
         InputStream stream = new ByteArrayInputStream("".getBytes());
         MessageReader reader =
-                new MessageReader().initialize(stream, shardId, new ObjectMapper(), Executors.newCachedThreadPool());
+                new MessageReader().initialize(stream, SHARD_ID, new ObjectMapper(), Executors.newCachedThreadPool());
         Future<Message> future = reader.getNextMessageFromSTDOUT();
 
         try {

amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/MessageWriterTest.java

@@ -0,0 +1,160 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package software.amazon.kinesis.multilang;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.mockito.Mockito;
+import software.amazon.kinesis.lifecycle.events.InitializationInput;
+import software.amazon.kinesis.lifecycle.events.LeaseLostInput;
+import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput;
+import software.amazon.kinesis.lifecycle.events.ShardEndedInput;
+import software.amazon.kinesis.multilang.messages.Message;
+import software.amazon.kinesis.retrieval.KinesisClientRecord;
+
+import static org.mockito.Mockito.verify;
+
+public class MessageWriterTest {
+
+    private static final String SHARD_ID = "shard-123";
+    MessageWriter messageWriter;
+    OutputStream stream;
+
+    @Rule
+    public final ExpectedException thrown = ExpectedException.none();
+
+    @Before
+    public void setup() {
+        stream = Mockito.mock(OutputStream.class);
+        messageWriter =
+                new MessageWriter().initialize(stream, SHARD_ID, new ObjectMapper(), Executors.newCachedThreadPool());
+    }
+
+    /*
+     * Here we are just testing that calling write causes bytes to get written to the stream.
+     */
+    @Test
+    public void writeCheckpointMessageNoErrorTest() throws IOException, InterruptedException, ExecutionException {
+        Future<Boolean> future = this.messageWriter.writeCheckpointMessageWithError("1234", 0L, null);
+        future.get();
+        verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt());
+        verify(this.stream, Mockito.atLeastOnce()).flush();
+    }
+
+    @Test
+    public void writeCheckpointMessageWithErrorTest() throws IOException, InterruptedException, ExecutionException {
+        Future<Boolean> future = this.messageWriter.writeCheckpointMessageWithError("1234", 0L, new Throwable());
+        future.get();
+        verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt());
+        verify(this.stream, Mockito.atLeastOnce()).flush();
+    }
+
+    @Test
+    public void writeInitializeMessageTest() throws IOException, InterruptedException, ExecutionException {
+        Future<Boolean> future = this.messageWriter.writeInitializeMessage(
+                InitializationInput.builder().shardId(SHARD_ID).build());
+        future.get();
+        verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt());
+        verify(this.stream, Mockito.atLeastOnce()).flush();
+    }
+
+    @Test
+    public void writeProcessRecordsMessageTest() throws IOException, InterruptedException, ExecutionException {
+        List<KinesisClientRecord> records = Arrays.asList(
+                KinesisClientRecord.builder()
+                        .data(ByteBuffer.wrap("kitten".getBytes()))
+                        .partitionKey("some cats")
+                        .sequenceNumber("357234807854789057805")
+                        .build(),
+                KinesisClientRecord.builder().build());
+        Future<Boolean> future = this.messageWriter.writeProcessRecordsMessage(
+                ProcessRecordsInput.builder().records(records).build());
+        future.get();
+
+        verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt());
+        verify(this.stream, Mockito.atLeastOnce()).flush();
+    }
+
+    @Test
+    public void writeShutdownMessageTest() throws IOException, InterruptedException, ExecutionException {
+        Future<Boolean> future = this.messageWriter.writeShardEndedMessage(
+                ShardEndedInput.builder().build());
+        future.get();
+
+        verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt());
+        verify(this.stream, Mockito.atLeastOnce()).flush();
+    }
+
+    @Test
+    public void writeShutdownRequestedMessageTest() throws IOException, InterruptedException, ExecutionException {
+        Future<Boolean> future = this.messageWriter.writeShutdownRequestedMessage();
+        future.get();
+
+        verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt());
+        verify(this.stream, Mockito.atLeastOnce()).flush();
+    }
+
+    @Test
+    public void streamIOExceptionTest() throws IOException, InterruptedException, ExecutionException {
+        Mockito.doThrow(IOException.class).when(stream).flush();
+        Future<Boolean> initializeTask = this.messageWriter.writeInitializeMessage(
+                InitializationInput.builder().shardId(SHARD_ID).build());
+        Boolean result = initializeTask.get();
+        Assert.assertNotNull(result);
+        Assert.assertFalse(result);
+    }
+
+    @Test
+    public void objectMapperFails() throws JsonProcessingException {
+        thrown.expect(RuntimeException.class);
+        thrown.expectMessage("Encountered I/O error while writing LeaseLostMessage action to subprocess");
+
+        ObjectMapper mapper = Mockito.mock(ObjectMapper.class);
+        Mockito.doThrow(JsonProcessingException.class).when(mapper).writeValueAsString(Mockito.any(Message.class));
+        messageWriter = new MessageWriter().initialize(stream, SHARD_ID, mapper, Executors.newCachedThreadPool());
+
+        messageWriter.writeLeaseLossMessage(LeaseLostInput.builder().build());
+    }
+
+    @Test
+    public void closeWriterTest() throws IOException {
+        Assert.assertTrue(this.messageWriter.isOpen());
+        this.messageWriter.close();
+        verify(this.stream, Mockito.times(1)).close();
+        Assert.assertFalse(this.messageWriter.isOpen());
+        try {
+            // Any message should fail
+            this.messageWriter.writeInitializeMessage(
+                    InitializationInput.builder().shardId(SHARD_ID).build());
+            Assert.fail("MessageWriter should be closed and unable to write.");
+        } catch (IllegalStateException e) {
+            // This should happen.
+        }
+    }
+}

amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/MultiLangDaemonConfigTest.java

@@ -0,0 +1,211 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package software.amazon.kinesis.multilang;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+
+import junit.framework.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.runners.MockitoJUnitRunner;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.kinesis.multilang.config.KinesisClientLibConfigurator;
+import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class MultiLangDaemonConfigTest {
+    private static final String FILENAME = "multilang.properties";
+    private static final String EXE = "TestExe.exe";
+    private static final String APPLICATION_NAME = MultiLangDaemonConfigTest.class.getSimpleName();
+    private static final String STREAM_NAME = "fakeStream";
+    private static final String STREAM_NAME_IN_ARN = "FAKE_STREAM_NAME";
+    private static final Region REGION = Region.US_EAST_1;
+    private static final String STREAM_ARN = "arn:aws:kinesis:us-east-2:012345678987:stream/" + STREAM_NAME_IN_ARN;
+
+    @Mock
+    private ClassLoader classLoader;
+
+    @Mock
+    private AwsCredentialsProvider credentialsProvider;
+
+    @Mock
+    private AwsCredentials creds;
+
+    private final KinesisClientLibConfigurator configurator = new KinesisClientLibConfigurator();
+    private MultiLangDaemonConfig deamonConfig;
+
+    /**
+     * Instantiate a MultiLangDaemonConfig object
+     * @param streamName
+     * @param streamArn
+     * @throws IOException
+     */
+    public void setup(String streamName, String streamArn) throws IOException {
+        String properties = String.format(
+                "executableName = %s\n"
+                        + "applicationName = %s\n"
+                        + "AwsCredentialsProvider = DefaultCredentialsProvider\n"
+                        + "processingLanguage = malbolge\n"
+                        + "regionName = %s\n",
+                EXE, APPLICATION_NAME, "us-east-1");
+
+        if (streamName != null) {
+            properties += String.format("streamName = %s\n", streamName);
+        }
+        if (streamArn != null) {
+            properties += String.format("streamArn = %s\n", streamArn);
+        }
+        classLoader = Mockito.mock(ClassLoader.class);
+
+        Mockito.doReturn(new ByteArrayInputStream(properties.getBytes()))
+                .when(classLoader)
+                .getResourceAsStream(FILENAME);
+
+        when(credentialsProvider.resolveCredentials()).thenReturn(creds);
+        when(creds.accessKeyId()).thenReturn("cool-user");
+        deamonConfig = new MultiLangDaemonConfig(FILENAME, classLoader, configurator);
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testConstructorFailsBecauseStreamArnIsInvalid() throws Exception {
+        setup("", "this_is_not_a_valid_arn");
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testConstructorFailsBecauseStreamArnIsInvalid2() throws Exception {
+        setup("", "arn:aws:kinesis:us-east-2:ACCOUNT_ID:BadFormatting:stream/" + STREAM_NAME_IN_ARN);
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testConstructorFailsBecauseStreamNameAndArnAreEmpty() throws Exception {
+        setup("", "");
+    }
+
+    @Test(expected = NullPointerException.class)
+    public void testConstructorFailsBecauseStreamNameAndArnAreNull() throws Exception {
+        setup(null, null);
+    }
+
+    @Test(expected = NullPointerException.class)
+    public void testConstructorFailsBecauseStreamNameIsNullAndArnIsEmpty() throws Exception {
+        setup(null, "");
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testConstructorFailsBecauseStreamNameIsEmptyAndArnIsNull() throws Exception {
+        setup("", null);
+    }
+
+    @Test
+    public void testConstructorUsingStreamName() throws IOException {
+        setup(STREAM_NAME, null);
+
+        assertConfigurationsMatch(STREAM_NAME, null);
+    }
+
+    @Test
+    public void testConstructorUsingStreamNameAndStreamArnIsEmpty() throws IOException {
+        setup(STREAM_NAME, "");
+
+        assertConfigurationsMatch(STREAM_NAME, "");
+    }
+
+    @Test
+    public void testConstructorUsingStreamNameAndStreamArnIsWhitespace() throws IOException {
+        setup(STREAM_NAME, "   ");
+
+        assertConfigurationsMatch(STREAM_NAME, "");
+    }
+
+    @Test
+    public void testConstructorUsingStreamArn() throws IOException {
+        setup(null, STREAM_ARN);
+
+        assertConfigurationsMatch(STREAM_NAME_IN_ARN, STREAM_ARN);
+    }
+
+    @Test
+    public void testConstructorUsingStreamNameAsEmptyAndStreamArn() throws IOException {
+        setup("", STREAM_ARN);
+
+        assertConfigurationsMatch(STREAM_NAME_IN_ARN, STREAM_ARN);
+    }
+
+    @Test
+    public void testConstructorUsingStreamArnOverStreamName() throws IOException {
+        setup(STREAM_NAME, STREAM_ARN);
+
+        assertConfigurationsMatch(STREAM_NAME_IN_ARN, STREAM_ARN);
+    }
+
+    /**
+     * Verify the daemonConfig properties are what we expect them to be.
+     *
+     * @param expectedStreamName
+     */
+    private void assertConfigurationsMatch(String expectedStreamName, String expectedStreamArn) {
+        final MultiLangDaemonConfiguration multiLangConfiguration = deamonConfig.getMultiLangDaemonConfiguration();
+        assertNotNull(deamonConfig.getExecutorService());
+        assertNotNull(multiLangConfiguration);
+        assertNotNull(deamonConfig.getRecordProcessorFactory());
+
+        assertEquals(EXE, deamonConfig.getRecordProcessorFactory().getCommandArray()[0]);
+        assertEquals(APPLICATION_NAME, multiLangConfiguration.getApplicationName());
+        assertEquals(expectedStreamName, multiLangConfiguration.getStreamName());
+        assertEquals(REGION, multiLangConfiguration.getDynamoDbClient().get("region"));
+        assertEquals(REGION, multiLangConfiguration.getCloudWatchClient().get("region"));
+        assertEquals(REGION, multiLangConfiguration.getKinesisClient().get("region"));
+        assertEquals(expectedStreamArn, multiLangConfiguration.getStreamArn());
+    }
+
+    @Test
+    public void testPropertyValidation() {
+        String propertiesNoExecutableName = "applicationName = testApp \n" + "streamName = fakeStream \n"
+                + "AwsCredentialsProvider = DefaultCredentialsProvider\n" + "processingLanguage = malbolge";
+        ClassLoader classLoader = Mockito.mock(ClassLoader.class);
+
+        Mockito.doReturn(new ByteArrayInputStream(propertiesNoExecutableName.getBytes()))
+                .when(classLoader)
+                .getResourceAsStream(FILENAME);
+
+        try {
+            new MultiLangDaemonConfig(FILENAME, classLoader, configurator);
+            Assert.fail("Construction of the config should have failed due to property validation failing.");
+        } catch (IllegalArgumentException e) {
+            // Good
+        } catch (IOException e) {
+            Assert.fail();
+        }
+    }
+
+    /**
+     * Test the loading of a "real" properties file. This test should catch
+     * any issues which might arise if there is a discrepancy between reality
+     * and mocking.
+     */
+    @Test
+    public void testActualPropertiesFile() throws Exception {
+        new MultiLangDaemonConfig(FILENAME);
+    }
+}

amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/MultiLangDaemonTest.java

@@ -0,0 +1,281 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package software.amazon.kinesis.multilang;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+
+import ch.qos.logback.classic.LoggerContext;
+import ch.qos.logback.classic.joran.JoranConfigurator;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.runners.MockitoJUnitRunner;
+import org.slf4j.LoggerFactory;
+import software.amazon.kinesis.coordinator.Scheduler;
+import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Matchers.anyObject;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class MultiLangDaemonTest {
+    @Mock
+    private Scheduler scheduler;
+
+    @Mock
+    private MultiLangDaemonConfig config;
+
+    @Mock
+    private ExecutorService executorService;
+
+    @Mock
+    private Future<Integer> futureInteger;
+
+    @Mock
+    private MultiLangDaemonConfiguration multiLangDaemonConfiguration;
+
+    @Mock
+    private Runtime runtime;
+
+    @Rule
+    public ExpectedException expectedException = ExpectedException.none();
+
+    @Rule
+    public final TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+    private MultiLangDaemon daemon;
+
+    @Before
+    public void setup() {
+        daemon = new MultiLangDaemon() {
+            @Override
+            Scheduler buildScheduler(final MultiLangDaemonConfig configuration) {
+                return scheduler;
+            }
+        };
+    }
+
+    @Test
+    public void testSuccessfulNoOptionsJCommanderBuild() {
+        String testPropertiesFile = "/test/properties/file";
+        MultiLangDaemon.MultiLangDaemonArguments arguments = new MultiLangDaemon.MultiLangDaemonArguments();
+        daemon.buildJCommanderAndParseArgs(arguments, new String[] {testPropertiesFile});
+
+        assertThat(arguments.propertiesFile, nullValue());
+        assertThat(arguments.logConfiguration, nullValue());
+        assertThat(arguments.parameters.size(), equalTo(1));
+        assertThat(arguments.parameters.get(0), equalTo(testPropertiesFile));
+    }
+
+    @Test
+    public void testSuccessfulOptionsJCommanderBuild() {
+        String propertiesOption = "/test/properties/file/option";
+        String propertiesFileArgs = "/test/properties/args";
+        String[] args = new String[] {"-p", propertiesOption, propertiesFileArgs};
+        MultiLangDaemon.MultiLangDaemonArguments arguments = new MultiLangDaemon.MultiLangDaemonArguments();
+        daemon.buildJCommanderAndParseArgs(arguments, args);
+
+        assertThat(arguments.propertiesFile, equalTo(propertiesOption));
+        assertThat(arguments.logConfiguration, nullValue());
+        assertThat(arguments.parameters.size(), equalTo(1));
+        assertThat(arguments.parameters.get(0), equalTo(propertiesFileArgs));
+    }
+
+    @Test
+    public void testEmptyArgsJCommanderBuild() {
+        MultiLangDaemon.MultiLangDaemonArguments arguments = new MultiLangDaemon.MultiLangDaemonArguments();
+        String[] args = new String[] {};
+        daemon.buildJCommanderAndParseArgs(arguments, args);
+
+        assertThat(arguments.propertiesFile, nullValue());
+        assertThat(arguments.logConfiguration, nullValue());
+        assertThat(arguments.parameters, empty());
+    }
+
+    @Test
+    public void testSuccessfulLoggingConfiguration() {
+        LoggerContext loggerContext = spy((LoggerContext) LoggerFactory.getILoggerFactory());
+        JoranConfigurator configurator = spy(new JoranConfigurator());
+
+        String logConfiguration =
+                this.getClass().getClassLoader().getResource("logback.xml").getPath();
+        daemon.configureLogging(logConfiguration, loggerContext, configurator);
+
+        verify(loggerContext).reset();
+        verify(configurator).setContext(eq(loggerContext));
+    }
+
+    @Test
+    public void testUnsuccessfulLoggingConfiguration() {
+        LoggerContext loggerContext = (LoggerContext) LoggerFactory.getILoggerFactory();
+        JoranConfigurator configurator = new JoranConfigurator();
+
+        expectedException.expect(RuntimeException.class);
+        expectedException.expectMessage(containsString("Error while loading log configuration:"));
+
+        String logConfiguration = "blahblahblah";
+
+        daemon.configureLogging(logConfiguration, loggerContext, configurator);
+    }
+
+    @Test
+    public void testNoPropertiesFileArgumentOrOption() {
+        expectedException.expect(RuntimeException.class);
+        expectedException.expectMessage(equalTo("Properties file missing, please provide a properties file"));
+
+        MultiLangDaemon.MultiLangDaemonArguments arguments = new MultiLangDaemon.MultiLangDaemonArguments();
+
+        daemon.validateAndGetPropertiesFileName(arguments);
+    }
+
+    @Test
+    public void testSuccessfulPropertiesArgument() {
+        String expectedPropertiesFile = "/test/properties/file";
+        MultiLangDaemon.MultiLangDaemonArguments arguments = new MultiLangDaemon.MultiLangDaemonArguments();
+        arguments.parameters = Collections.singletonList(expectedPropertiesFile);
+
+        String propertiesFile = daemon.validateAndGetPropertiesFileName(arguments);
+
+        assertThat(propertiesFile, equalTo(expectedPropertiesFile));
+    }
+
+    @Test
+    public void testPropertiesOptionsOverrideArgument() {
+        String propertiesArgument = "/test/properties/argument";
+        String propertiesOptions = "/test/properties/options";
+
+        MultiLangDaemon.MultiLangDaemonArguments arguments = new MultiLangDaemon.MultiLangDaemonArguments();
+        arguments.parameters = Collections.singletonList(propertiesArgument);
+        arguments.propertiesFile = propertiesOptions;
+
+        String propertiesFile = daemon.validateAndGetPropertiesFileName(arguments);
+
+        assertThat(propertiesFile, equalTo(propertiesOptions));
+    }
+
+    @Test
+    public void testExtraArgumentsFailure() {
+        expectedException.expect(RuntimeException.class);
+        expectedException.expectMessage(containsString("Expected a single argument, but found multiple arguments."));
+
+        MultiLangDaemon.MultiLangDaemonArguments arguments = new MultiLangDaemon.MultiLangDaemonArguments();
+        arguments.parameters = Arrays.asList("parameter1", "parameter2");
+
+        daemon.validateAndGetPropertiesFileName(arguments);
+    }
+
+    @Test
+    public void testBuildMultiLangConfigMissingPropertiesFile() {
+        expectedException.expect(RuntimeException.class);
+        expectedException.expectMessage(containsString("Error while reading properties file:"));
+
+        daemon.buildMultiLangDaemonConfig("blahblahblah");
+    }
+
+    @Test
+    public void testBuildMultiLangConfigWithIncorrectInformation() throws IOException {
+        File propertiesFile = temporaryFolder.newFile("temp.properties");
+
+        expectedException.expect(RuntimeException.class);
+        expectedException.expectMessage(containsString("Must provide an executable name in the properties file"));
+
+        daemon.buildMultiLangDaemonConfig(propertiesFile.getAbsolutePath());
+    }
+
+    @Test
+    public void testSuccessfulSubmitRunnerAndWait() throws Exception {
+        int expectedExitCode = 0;
+
+        MultiLangDaemon.MultiLangRunner runner = new MultiLangDaemon.MultiLangRunner(scheduler);
+        when(config.getExecutorService()).thenReturn(executorService);
+        when(executorService.submit(eq(runner))).thenReturn(futureInteger);
+        when(futureInteger.get()).thenReturn(expectedExitCode);
+
+        int exitCode = daemon.submitRunnerAndWait(config, runner);
+
+        assertThat(exitCode, equalTo(expectedExitCode));
+    }
+
+    @Test
+    public void testErrorSubmitRunnerAndWait() throws Exception {
+        int expectedExitCode = 1;
+
+        MultiLangDaemon.MultiLangRunner runner = new MultiLangDaemon.MultiLangRunner(scheduler);
+        when(config.getExecutorService()).thenReturn(executorService);
+        when(executorService.submit(eq(runner))).thenReturn(futureInteger);
+        when(futureInteger.get()).thenThrow(ExecutionException.class);
+
+        int exitCode = daemon.submitRunnerAndWait(config, runner);
+
+        assertThat(exitCode, equalTo(expectedExitCode));
+    }
+
+    @Test
+    public void testSetupShutdownHook() {
+        when(config.getMultiLangDaemonConfiguration()).thenReturn(multiLangDaemonConfiguration);
+        when(multiLangDaemonConfiguration.getShutdownGraceMillis()).thenReturn(1000L);
+        doNothing().when(runtime).addShutdownHook(anyObject());
+
+        MultiLangDaemon.MultiLangRunner runner = new MultiLangDaemon.MultiLangRunner(scheduler);
+        daemon.setupShutdownHook(runtime, runner, config);
+
+        verify(multiLangDaemonConfiguration).getShutdownGraceMillis();
+        verify(runtime).addShutdownHook(anyObject());
+    }
+
+    @Test
+    public void testSuccessfulRunner() throws Exception {
+        MultiLangDaemon.MultiLangRunner runner = new MultiLangDaemon.MultiLangRunner(scheduler);
+        doNothing().when(scheduler).run();
+
+        int exit = runner.call();
+
+        assertThat(exit, equalTo(0));
+
+        verify(scheduler).run();
+    }
+
+    @Test
+    public void testUnsuccessfulRunner() throws Exception {
+        MultiLangDaemon.MultiLangRunner runner = new MultiLangDaemon.MultiLangRunner(scheduler);
+        doThrow(Exception.class).when(scheduler).run();
+
+        int exit = runner.call();
+
+        assertThat(exit, equalTo(1));
+
+        verify(scheduler).run();
+    }
+}

amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/MultiLangProtocolTest.java

@@ -0,0 +1,290 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package software.amazon.kinesis.multilang;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import com.google.common.util.concurrent.SettableFuture;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.runners.MockitoJUnitRunner;
+import org.mockito.stubbing.Answer;
+import software.amazon.kinesis.exceptions.InvalidStateException;
+import software.amazon.kinesis.exceptions.KinesisClientLibDependencyException;
+import software.amazon.kinesis.exceptions.ShutdownException;
+import software.amazon.kinesis.exceptions.ThrottlingException;
+import software.amazon.kinesis.lifecycle.events.InitializationInput;
+import software.amazon.kinesis.lifecycle.events.LeaseLostInput;
+import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput;
+import software.amazon.kinesis.lifecycle.events.ShardEndedInput;
+import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
+import software.amazon.kinesis.multilang.messages.CheckpointMessage;
+import software.amazon.kinesis.multilang.messages.LeaseLostMessage;
+import software.amazon.kinesis.multilang.messages.Message;
+import software.amazon.kinesis.multilang.messages.ProcessRecordsMessage;
+import software.amazon.kinesis.multilang.messages.ShardEndedMessage;
+import software.amazon.kinesis.multilang.messages.StatusMessage;
+import software.amazon.kinesis.processor.RecordProcessorCheckpointer;
+import software.amazon.kinesis.retrieval.KinesisClientRecord;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyInt;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.timeout;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class MultiLangProtocolTest {
+    private static final List<KinesisClientRecord> EMPTY_RECORD_LIST = Collections.emptyList();
+
+    @Mock
+    private MultiLangProtocol protocol;
+
+    @Mock
+    private MessageWriter messageWriter;
+
+    @Mock
+    private MessageReader messageReader;
+
+    private String shardId;
+
+    @Mock
+    private RecordProcessorCheckpointer checkpointer;
+
+    @Mock
+    private MultiLangDaemonConfiguration configuration;
+
+    @Before
+    public void setup() {
+        this.shardId = "shard-id-123";
+        protocol = new MultiLangProtocolForTesting(
+                messageReader,
+                messageWriter,
+                InitializationInput.builder().shardId(shardId).build(),
+                configuration);
+
+        when(configuration.getTimeoutInSeconds()).thenReturn(null);
+    }
+
+    private <T> Future<T> buildFuture(T value) {
+        SettableFuture<T> future = SettableFuture.create();
+        future.set(value);
+        return future;
+    }
+
+    private <T> Future<T> buildFuture(T value, Class<T> clazz) {
+        SettableFuture<T> future = SettableFuture.create();
+        future.set(value);
+        return future;
+    }
+
+    @Test
+    public void testInitialize() {
+        when(messageWriter.writeInitializeMessage(argThat(Matchers.withInit(
+                        InitializationInput.builder().shardId(shardId).build()))))
+                .thenReturn(buildFuture(true));
+        when(messageReader.getNextMessageFromSTDOUT())
+                .thenReturn(buildFuture(new StatusMessage("initialize"), Message.class));
+        assertThat(protocol.initialize(), equalTo(true));
+    }
+
+    @Test
+    public void testProcessRecords() {
+        when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class)))
+                .thenReturn(buildFuture(true));
+        when(messageReader.getNextMessageFromSTDOUT())
+                .thenReturn(buildFuture(new StatusMessage("processRecords"), Message.class));
+
+        assertThat(
+                protocol.processRecords(
+                        ProcessRecordsInput.builder().records(EMPTY_RECORD_LIST).build()),
+                equalTo(true));
+    }
+
+    @Test
+    public void leaseLostTest() {
+        when(messageWriter.writeLeaseLossMessage(any(LeaseLostInput.class))).thenReturn(buildFuture(true));
+        when(messageReader.getNextMessageFromSTDOUT())
+                .thenReturn(buildFuture(new StatusMessage(LeaseLostMessage.ACTION), Message.class));
+
+        assertThat(protocol.leaseLost(LeaseLostInput.builder().build()), equalTo(true));
+    }
+
+    @Test
+    public void shardEndedTest() {
+        when(messageWriter.writeShardEndedMessage(any(ShardEndedInput.class))).thenReturn(buildFuture(true));
+        when(messageReader.getNextMessageFromSTDOUT())
+                .thenReturn(buildFuture(new StatusMessage(ShardEndedMessage.ACTION)));
+
+        assertThat(protocol.shardEnded(ShardEndedInput.builder().build()), equalTo(true));
+    }
+
+    @Test
+    public void shutdownRequestedTest() {
+        when(messageWriter.writeShutdownRequestedMessage()).thenReturn(buildFuture(true));
+        when(messageReader.getNextMessageFromSTDOUT())
+                .thenReturn(buildFuture(new StatusMessage("shutdownRequested"), Message.class));
+        Mockito.doReturn(buildFuture(true)).when(messageWriter).writeShutdownRequestedMessage();
+        Mockito.doReturn(buildFuture(new StatusMessage("shutdownRequested")))
+                .when(messageReader)
+                .getNextMessageFromSTDOUT();
+        assertThat(protocol.shutdownRequested(null), equalTo(true));
+    }
+
+    private Answer<Future<Message>> buildMessageAnswers(List<Message> messages) {
+        return new Answer<Future<Message>>() {
+
+            Iterator<Message> messageIterator;
+            Message message;
+
+            Answer<Future<Message>> init(List<Message> messages) {
+                messageIterator = messages.iterator();
+                return this;
+            }
+
+            @Override
+            public Future<Message> answer(InvocationOnMock invocation) throws Throwable {
+                if (this.messageIterator.hasNext()) {
+                    message = this.messageIterator.next();
+                }
+                return buildFuture(message);
+            }
+        }.init(messages);
+    }
+
+    @Test
+    public void testProcessRecordsWithCheckpoints()
+            throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
+
+        when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class)))
+                .thenReturn(buildFuture(true));
+        when(messageWriter.writeCheckpointMessageWithError(anyString(), anyLong(), any(Throwable.class)))
+                .thenReturn(buildFuture(true));
+        when(messageReader.getNextMessageFromSTDOUT()).thenAnswer(buildMessageAnswers(new ArrayList<Message>() {
+            {
+                this.add(new CheckpointMessage("123", 0L, null));
+                this.add(new CheckpointMessage(null, 0L, null));
+                /*
+                 * This procesRecords message will be ignored by the read loop which only cares about status and
+                 * checkpoint messages. All other lines and message types are ignored. By inserting it here, we check
+                 * that this test succeeds even with unexpected messaging.
+                 */
+                this.add(new ProcessRecordsMessage());
+                this.add(new StatusMessage("processRecords"));
+            }
+        }));
+
+        boolean result = protocol.processRecords(ProcessRecordsInput.builder()
+                .records(EMPTY_RECORD_LIST)
+                .checkpointer(checkpointer)
+                .build());
+
+        assertThat(result, equalTo(true));
+
+        verify(checkpointer, timeout(1)).checkpoint();
+        verify(checkpointer, timeout(1)).checkpoint("123", 0L);
+    }
+
+    @Test
+    public void testProcessRecordsWithABadCheckpoint() {
+        when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class)))
+                .thenReturn(buildFuture(true));
+        when(messageWriter.writeCheckpointMessageWithError(anyString(), anyLong(), any(Throwable.class)))
+                .thenReturn(buildFuture(false));
+        when(messageReader.getNextMessageFromSTDOUT()).thenAnswer(buildMessageAnswers(new ArrayList<Message>() {
+            {
+                this.add(new CheckpointMessage("456", 0L, null));
+                this.add(new StatusMessage("processRecords"));
+            }
+        }));
+        assertThat(
+                protocol.processRecords(ProcessRecordsInput.builder()
+                        .records(EMPTY_RECORD_LIST)
+                        .checkpointer(checkpointer)
+                        .build()),
+                equalTo(false));
+    }
+
+    @Test(expected = NullPointerException.class)
+    public void waitForStatusMessageTimeoutTest() throws InterruptedException, TimeoutException, ExecutionException {
+        when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class)))
+                .thenReturn(buildFuture(true));
+        Future<Message> future = Mockito.mock(Future.class);
+        when(messageReader.getNextMessageFromSTDOUT()).thenReturn(future);
+        when(configuration.getTimeoutInSeconds()).thenReturn(5);
+        when(future.get(anyInt(), eq(TimeUnit.SECONDS))).thenThrow(TimeoutException.class);
+        protocol = new MultiLangProtocolForTesting(
+                messageReader,
+                messageWriter,
+                InitializationInput.builder().shardId(shardId).build(),
+                configuration);
+
+        protocol.processRecords(
+                ProcessRecordsInput.builder().records(EMPTY_RECORD_LIST).build());
+    }
+
+    @Test
+    public void waitForStatusMessageSuccessTest() {
+        when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class)))
+                .thenReturn(buildFuture(true));
+        when(messageReader.getNextMessageFromSTDOUT())
+                .thenReturn(buildFuture(new StatusMessage("processRecords"), Message.class));
+        when(configuration.getTimeoutInSeconds()).thenReturn(5);
+
+        assertTrue(protocol.processRecords(
+                ProcessRecordsInput.builder().records(EMPTY_RECORD_LIST).build()));
+    }
+
+    private class MultiLangProtocolForTesting extends MultiLangProtocol {
+        /**
+         * Constructor.
+         *
+         * @param messageReader       A message reader.
+         * @param messageWriter       A message writer.
+         * @param initializationInput
+         * @param configuration
+         */
+        MultiLangProtocolForTesting(
+                final MessageReader messageReader,
+                final MessageWriter messageWriter,
+                final InitializationInput initializationInput,
+                final MultiLangDaemonConfiguration configuration) {
+            super(messageReader, messageWriter, initializationInput, configuration);
+        }
+
+        @Override
+        protected void haltJvm(final int exitStatus) {
+            throw new NullPointerException();
+        }
+    }
+}

amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/NestedPropertyKeyTest.java

@@ -0,0 +1,110 @@
+/*
+ * Copyright 2023 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package software.amazon.kinesis.multilang;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.runners.MockitoJUnitRunner;
+import software.amazon.awssdk.regions.Region;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyZeroInteractions;
+import static software.amazon.kinesis.multilang.NestedPropertyKey.ENDPOINT;
+import static software.amazon.kinesis.multilang.NestedPropertyKey.ENDPOINT_REGION;
+import static software.amazon.kinesis.multilang.NestedPropertyKey.EXTERNAL_ID;
+import static software.amazon.kinesis.multilang.NestedPropertyKey.parse;
+
+@RunWith(MockitoJUnitRunner.class)
+public class NestedPropertyKeyTest {
+
+    @Mock
+    private NestedPropertyProcessor mockProcessor;
+
+    @Test
+    public void testExternalId() {
+        final String expectedId = "eid";
+
+        parse(mockProcessor, createKey(EXTERNAL_ID, expectedId));
+        verify(mockProcessor).acceptExternalId(expectedId);
+    }
+
+    @Test
+    public void testEndpoint() {
+        final String expectedEndpoint = "https://sts.us-east-1.amazonaws.com";
+        final String expectedRegion = "us-east-1";
+        final String param = createKey(ENDPOINT, expectedEndpoint + "^" + expectedRegion);
+
+        parse(mockProcessor, param);
+        verify(mockProcessor).acceptEndpoint(expectedEndpoint, expectedRegion);
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testInvalidEndpoint() {
+        parse(mockProcessor, createKey(ENDPOINT, "value-sans-caret-delimiter"));
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testInvalidEndpointDoubleCaret() {
+        parse(mockProcessor, createKey(ENDPOINT, "https://sts.us-east-1.amazonaws.com^us-east-1^borkbork"));
+    }
+
+    @Test
+    public void testEndpointRegion() {
+        final Region expectedRegion = Region.US_GOV_WEST_1;
+
+        parse(mockProcessor, createKey(ENDPOINT_REGION, expectedRegion.id()));
+        verify(mockProcessor).acceptEndpointRegion(expectedRegion);
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testInvalidEndpointRegion() {
+        parse(mockProcessor, createKey(ENDPOINT_REGION, "snuffleupagus"));
+    }
+
+    /**
+     * Test that the literal nested key (i.e., {@code key=} in {@code some_val|key=nested_val})
+     * does not change. Any change to an existing literal key is not backwards-compatible.
+     */
+    @Test
+    public void testKeysExplicitly() {
+        // Adding a new enum will deliberately cause this assert to fail, and
+        // therefore raise awareness for this explicit test. Add-and-remove may
+        // keep the number unchanged yet will also break (by removing an enum).
+        assertEquals(3, NestedPropertyKey.values().length);
+
+        assertEquals("endpoint", ENDPOINT.getNestedKey());
+        assertEquals("endpointRegion", ENDPOINT_REGION.getNestedKey());
+        assertEquals("externalId", EXTERNAL_ID.getNestedKey());
+    }
+
+    @Test
+    public void testNonmatchingParameters() {
+        final String[] params = new String[] {
+            null,
+            "",
+            "hello world", // no nested key
+            "foo=bar", // nested key, but is not a recognized key
+            createKey(EXTERNAL_ID, "eid") + "=extra", // valid key made invalid by second '='
+        };
+        parse(mockProcessor, params);
+        verifyZeroInteractions(mockProcessor);
+    }
+
+    private static String createKey(final NestedPropertyKey key, final String value) {
+        return key.getNestedKey() + "=" + value;
+    }
+}

amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/ReadSTDERRTaskTest.java

@@ -1,18 +1,18 @@
 /*
- * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
-package com.amazonaws.services.kinesis.multilang;
+package software.amazon.kinesis.multilang;
 
 import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
@@ -30,7 +30,7 @@ import org.mockito.Mockito;
 
 public class ReadSTDERRTaskTest {
 
-    private static final String shardId = "shard-123";
+    private static final String SHARD_ID = "shard-123";
     private BufferedReader mockBufferReader;
 
     @Before
@@ -43,7 +43,7 @@ public class ReadSTDERRTaskTest {
 
         String errorMessages = "OMG\nThis is test message\n blah blah blah \n";
         InputStream stream = new ByteArrayInputStream(errorMessages.getBytes());
-        LineReaderTask<Boolean> reader = new DrainChildSTDERRTask().initialize(stream, shardId, "");
+        LineReaderTask<Boolean> reader = new DrainChildSTDERRTask().initialize(stream, SHARD_ID, "");
         Assert.assertNotNull(reader);
     }
 
@@ -52,7 +52,7 @@ public class ReadSTDERRTaskTest {
         String errorMessages = "OMG\nThis is test message\n blah blah blah \n";
         BufferedReader bufferReader =
                 new BufferedReader(new InputStreamReader(new ByteArrayInputStream(errorMessages.getBytes())));
-        LineReaderTask<Boolean> errorReader = new DrainChildSTDERRTask().initialize(bufferReader, shardId, "");
+        LineReaderTask<Boolean> errorReader = new DrainChildSTDERRTask().initialize(bufferReader, SHARD_ID, "");
         Assert.assertNotNull(errorReader);
 
         Boolean result = errorReader.call();
@@ -65,14 +65,15 @@ public class ReadSTDERRTaskTest {
         } catch (IOException e) {
             Assert.fail("Not supposed to get an exception when we're just building our mock.");
         }
-        LineReaderTask<Boolean> errorReader = new DrainChildSTDERRTask().initialize(mockBufferReader, shardId, "");
+        LineReaderTask<Boolean> errorReader = new DrainChildSTDERRTask().initialize(mockBufferReader, SHARD_ID, "");
         Assert.assertNotNull(errorReader);
         Future<Boolean> result = Executors.newCachedThreadPool().submit(errorReader);
         Boolean finishedCleanly = null;
         try {
             finishedCleanly = result.get();
         } catch (InterruptedException | ExecutionException e) {
-            Assert.fail("Should have been able to get a result. The error should be handled during the call and result in false.");
+            Assert.fail(
+                    "Should have been able to get a result. The error should be handled during the call and result in false.");
         }
         Assert.assertFalse("Reading a line should have thrown an exception", finishedCleanly);
     }

amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/StreamingShardRecordProcessorFactoryTest.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package software.amazon.kinesis.multilang;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.runners.MockitoJUnitRunner;
+import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
+import software.amazon.kinesis.processor.ShardRecordProcessor;
+
+@RunWith(MockitoJUnitRunner.class)
+public class StreamingShardRecordProcessorFactoryTest {
+
+    @Mock
+    private MultiLangDaemonConfiguration configuration;
+
+    @Test
+    public void createProcessorTest() {
+        MultiLangRecordProcessorFactory factory =
+                new MultiLangRecordProcessorFactory("somecommand", null, configuration);
+        ShardRecordProcessor processor = factory.shardRecordProcessor();
+
+        Assert.assertEquals(
+                "Should have constructed a StreamingRecordProcessor",
+                MultiLangShardRecordProcessor.class,
+                processor.getClass());
+    }
+}

amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/StreamingShardRecordProcessorTest.java

@@ -0,0 +1,315 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package software.amazon.kinesis.multilang;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.runners.MockitoJUnitRunner;
+import org.mockito.stubbing.Answer;
+import software.amazon.awssdk.services.kinesis.model.Record;
+import software.amazon.kinesis.exceptions.KinesisClientLibDependencyException;
+import software.amazon.kinesis.exceptions.ThrottlingException;
+import software.amazon.kinesis.lifecycle.events.InitializationInput;
+import software.amazon.kinesis.lifecycle.events.LeaseLostInput;
+import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput;
+import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
+import software.amazon.kinesis.multilang.messages.InitializeMessage;
+import software.amazon.kinesis.multilang.messages.Message;
+import software.amazon.kinesis.multilang.messages.ProcessRecordsMessage;
+import software.amazon.kinesis.multilang.messages.ShutdownMessage;
+import software.amazon.kinesis.multilang.messages.StatusMessage;
+import software.amazon.kinesis.processor.Checkpointer;
+import software.amazon.kinesis.processor.PreparedCheckpointer;
+import software.amazon.kinesis.processor.RecordProcessorCheckpointer;
+import software.amazon.kinesis.retrieval.KinesisClientRecord;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class StreamingShardRecordProcessorTest {
+
+    private static final String SHARD_ID = "shard-123";
+
+    private int systemExitCount = 0;
+
+    @Mock
+    private Future<Message> messageFuture;
+
+    @Mock
+    private Future<Boolean> trueFuture;
+
+    private RecordProcessorCheckpointer unimplementedCheckpointer = new RecordProcessorCheckpointer() {
+
+        @Override
+        public void checkpoint() throws KinesisClientLibDependencyException, ThrottlingException {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public void checkpoint(String sequenceNumber)
+                throws KinesisClientLibDependencyException, ThrottlingException, IllegalArgumentException {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public void checkpoint(Record record) throws KinesisClientLibDependencyException, ThrottlingException {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public void checkpoint(String sequenceNumber, long subSequenceNumber)
+                throws KinesisClientLibDependencyException, ThrottlingException, IllegalArgumentException {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public PreparedCheckpointer prepareCheckpoint()
+                throws KinesisClientLibDependencyException, ThrottlingException {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public PreparedCheckpointer prepareCheckpoint(byte[] applicationState)
+                throws KinesisClientLibDependencyException, ThrottlingException {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public PreparedCheckpointer prepareCheckpoint(Record record)
+                throws KinesisClientLibDependencyException, ThrottlingException {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public PreparedCheckpointer prepareCheckpoint(Record record, byte[] applicationState)
+                throws KinesisClientLibDependencyException, ThrottlingException {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public PreparedCheckpointer prepareCheckpoint(String sequenceNumber)
+                throws KinesisClientLibDependencyException, ThrottlingException, IllegalArgumentException {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public PreparedCheckpointer prepareCheckpoint(String sequenceNumber, byte[] applicationState)
+                throws KinesisClientLibDependencyException, ThrottlingException, IllegalArgumentException {
+            return null;
+        }
+
+        @Override
+        public PreparedCheckpointer prepareCheckpoint(String sequenceNumber, long subSequenceNumber)
+                throws KinesisClientLibDependencyException, ThrottlingException, IllegalArgumentException {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public PreparedCheckpointer prepareCheckpoint(
+                String sequenceNumber, long subSequenceNumber, byte[] applicationState)
+                throws KinesisClientLibDependencyException, ThrottlingException, IllegalArgumentException {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public Checkpointer checkpointer() {
+            throw new UnsupportedOperationException();
+        }
+    };
+
+    private MessageWriter messageWriter;
+
+    private DrainChildSTDERRTask errorReader;
+
+    private MessageReader messageReader;
+
+    private MultiLangShardRecordProcessor recordProcessor;
+
+    @Mock
+    private MultiLangDaemonConfiguration configuration;
+
+    @Before
+    public void prepare() throws InterruptedException, ExecutionException {
+        // Fake command
+        systemExitCount = 0;
+
+        // Mocks
+        ExecutorService executor = Executors.newFixedThreadPool(3);
+        final Process process = Mockito.mock(Process.class);
+
+        messageWriter = Mockito.mock(MessageWriter.class);
+        messageReader = Mockito.mock(MessageReader.class);
+        errorReader = Mockito.mock(DrainChildSTDERRTask.class);
+        when(configuration.getTimeoutInSeconds()).thenReturn(null);
+
+        recordProcessor =
+                new MultiLangShardRecordProcessor(
+                        new ProcessBuilder(),
+                        executor,
+                        new ObjectMapper(),
+                        messageWriter,
+                        messageReader,
+                        errorReader,
+                        configuration) {
+
+                    // Just don't do anything when we exit.
+                    void exit() {
+                        systemExitCount += 1;
+                    }
+
+                    // Inject our mock process
+                    Process startProcess() {
+                        return process;
+                    }
+                };
+
+        // Our process will return mock streams
+        InputStream inputStream = Mockito.mock(InputStream.class);
+        InputStream errorStream = Mockito.mock(InputStream.class);
+        OutputStream outputStream = Mockito.mock(OutputStream.class);
+        Mockito.doReturn(inputStream).when(process).getInputStream();
+        Mockito.doReturn(errorStream).when(process).getErrorStream();
+        Mockito.doReturn(outputStream).when(process).getOutputStream();
+
+        Mockito.doReturn(Mockito.mock(Future.class)).when(messageReader).drainSTDOUT();
+        Mockito.doReturn(true).when(trueFuture).get();
+
+        when(messageWriter.writeInitializeMessage(any(InitializationInput.class)))
+                .thenReturn(trueFuture);
+        when(messageWriter.writeCheckpointMessageWithError(anyString(), anyLong(), any(Throwable.class)))
+                .thenReturn(trueFuture);
+        when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class)))
+                .thenReturn(trueFuture);
+        when(messageWriter.writeLeaseLossMessage(any(LeaseLostInput.class))).thenReturn(trueFuture);
+    }
+
+    private void phases(Answer<StatusMessage> answer) throws InterruptedException, ExecutionException {
+        /*
+         * Return a status message for each call
+         * Plan is:
+         * initialize
+         * processRecords
+         * processRecords
+         * shutdown
+         */
+        when(messageFuture.get()).thenAnswer(answer);
+        when(messageReader.getNextMessageFromSTDOUT()).thenReturn(messageFuture);
+
+        List<KinesisClientRecord> testRecords = Collections.emptyList();
+
+        recordProcessor.initialize(
+                InitializationInput.builder().shardId(SHARD_ID).build());
+        recordProcessor.processRecords(ProcessRecordsInput.builder()
+                .records(testRecords)
+                .checkpointer(unimplementedCheckpointer)
+                .build());
+        recordProcessor.processRecords(ProcessRecordsInput.builder()
+                .records(testRecords)
+                .checkpointer(unimplementedCheckpointer)
+                .build());
+        recordProcessor.leaseLost(LeaseLostInput.builder().build());
+    }
+
+    @Test
+    public void processorPhasesTest() throws InterruptedException, ExecutionException {
+        Answer<StatusMessage> answer = new Answer<StatusMessage>() {
+
+            StatusMessage[] answers = new StatusMessage[] {
+                new StatusMessage(InitializeMessage.ACTION),
+                new StatusMessage(ProcessRecordsMessage.ACTION),
+                new StatusMessage(ProcessRecordsMessage.ACTION),
+                new StatusMessage(ShutdownMessage.ACTION)
+            };
+
+            int callCount = 0;
+
+            @Override
+            public StatusMessage answer(InvocationOnMock invocation) throws Throwable {
+                if (callCount < answers.length) {
+                    return answers[callCount++];
+                } else {
+                    throw new Throwable("Too many calls to getNextStatusMessage");
+                }
+            }
+        };
+
+        phases(answer);
+
+        verify(messageWriter)
+                .writeInitializeMessage(argThat(Matchers.withInit(
+                        InitializationInput.builder().shardId(SHARD_ID).build())));
+        verify(messageWriter, times(2)).writeProcessRecordsMessage(any(ProcessRecordsInput.class));
+        verify(messageWriter).writeLeaseLossMessage(any(LeaseLostInput.class));
+    }
+
+    @Test
+    public void initFailsTest() throws InterruptedException, ExecutionException {
+        Answer<StatusMessage> answer = new Answer<StatusMessage>() {
+
+            /*
+             * This bad message will cause shutdown to not attempt to send a message. i.e. avoid encountering an
+             * exception.
+             */
+            StatusMessage[] answers = new StatusMessage[] {
+                new StatusMessage("Bad"),
+                new StatusMessage(ProcessRecordsMessage.ACTION),
+                new StatusMessage(ProcessRecordsMessage.ACTION),
+                new StatusMessage(ShutdownMessage.ACTION)
+            };
+
+            int callCount = 0;
+
+            @Override
+            public StatusMessage answer(InvocationOnMock invocation) throws Throwable {
+                if (callCount < answers.length) {
+                    return answers[callCount++];
+                } else {
+                    throw new Throwable("Too many calls to getNextStatusMessage");
+                }
+            }
+        };
+
+        phases(answer);
+
+        verify(messageWriter)
+                .writeInitializeMessage(argThat(Matchers.withInit(
+                        InitializationInput.builder().shardId(SHARD_ID).build())));
+        verify(messageWriter, times(2)).writeProcessRecordsMessage(any(ProcessRecordsInput.class));
+        verify(messageWriter, never()).writeLeaseLossMessage(any(LeaseLostInput.class));
+        Assert.assertEquals(1, systemExitCount);
+    }
+}

amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/auth/KclStsAssumeRoleCredentialsProviderTest.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright 2023 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package software.amazon.kinesis.multilang.auth;
+
+import java.util.Arrays;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class KclStsAssumeRoleCredentialsProviderTest {
+
+    private static final String ARN = "arn";
+    private static final String SESSION_NAME = "sessionName";
+
+    /**
+     * Test that the constructor doesn't throw an out-of-bounds exception if
+     * there are no parameters beyond the required ARN and session name.
+     */
+    @Test
+    public void testConstructorWithoutOptionalParams() {
+        new KclStsAssumeRoleCredentialsProvider(new String[] {ARN, SESSION_NAME, "endpointRegion=us-east-1"});
+    }
+
+    @Test
+    public void testAcceptEndpoint() {
+        // discovered exception during e2e testing; therefore, this test is
+        // to simply verify the constructed STS client doesn't go *boom*
+        final KclStsAssumeRoleCredentialsProvider provider =
+                new KclStsAssumeRoleCredentialsProvider(ARN, SESSION_NAME, "endpointRegion=us-east-1");
+        provider.acceptEndpoint("endpoint", "us-east-1");
+    }
+
+    @Test
+    public void testVarArgs() {
+        for (final String[] varargs : Arrays.asList(
+                new String[] {ARN, SESSION_NAME, "externalId=eid", "foo", "endpointRegion=us-east-1"},
+                new String[] {ARN, SESSION_NAME, "foo", "externalId=eid", "endpointRegion=us-east-1"})) {
+            final VarArgsSpy provider = new VarArgsSpy(varargs);
+            assertEquals("eid", provider.externalId);
+        }
+    }
+
+    private static class VarArgsSpy extends KclStsAssumeRoleCredentialsProvider {
+
+        private String externalId;
+
+        public VarArgsSpy(String[] args) {
+            super(args);
+        }
+
+        @Override
+        public void acceptExternalId(final String externalId) {
+            this.externalId = externalId;
+            super.acceptExternalId(externalId);
+        }
+    }
+}

amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/AwsCredentialsProviderPropertyValueDecoderTest.java

@@ -0,0 +1,237 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package software.amazon.kinesis.multilang.config;
+
+import java.util.Arrays;
+
+import lombok.ToString;
+import org.hamcrest.Description;
+import org.hamcrest.Matcher;
+import org.hamcrest.TypeSafeDiagnosingMatcher;
+import org.junit.Test;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain;
+import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider;
+import software.amazon.kinesis.multilang.auth.KclStsAssumeRoleCredentialsProvider;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThat;
+
+public class AwsCredentialsProviderPropertyValueDecoderTest {
+
+    private static final String TEST_ACCESS_KEY_ID = "123";
+    private static final String TEST_SECRET_KEY = "456";
+
+    private final String credentialName1 = AlwaysSucceedCredentialsProvider.class.getName();
+    private final String credentialName2 = ConstructorCredentialsProvider.class.getName();
+    private final String createCredentialClass = CreateProvider.class.getName();
+    private final AwsCredentialsProviderPropertyValueDecoder decoder = new AwsCredentialsProviderPropertyValueDecoder();
+
+    @ToString
+    private static class AwsCredentialsMatcher extends TypeSafeDiagnosingMatcher<AwsCredentialsProvider> {
+
+        private final Matcher<String> akidMatcher;
+        private final Matcher<String> secretMatcher;
+        private final Matcher<Class<?>> classMatcher;
+
+        public AwsCredentialsMatcher(String akid, String secret) {
+            this.akidMatcher = equalTo(akid);
+            this.secretMatcher = equalTo(secret);
+            this.classMatcher = instanceOf(AwsCredentialsProviderChain.class);
+        }
+
+        @Override
+        protected boolean matchesSafely(AwsCredentialsProvider item, Description mismatchDescription) {
+            AwsCredentials actual = item.resolveCredentials();
+            boolean matched = true;
+
+            if (!classMatcher.matches(item)) {
+                classMatcher.describeMismatch(item, mismatchDescription);
+                matched = false;
+            }
+
+            if (!akidMatcher.matches(actual.accessKeyId())) {
+                akidMatcher.describeMismatch(actual.accessKeyId(), mismatchDescription);
+                matched = false;
+            }
+            if (!secretMatcher.matches(actual.secretAccessKey())) {
+                secretMatcher.describeMismatch(actual.secretAccessKey(), mismatchDescription);
+                matched = false;
+            }
+            return matched;
+        }
+
+        @Override
+        public void describeTo(Description description) {
+            description
+                    .appendText("An AwsCredentialsProvider that provides an AwsCredential matching: ")
+                    .appendList("(", ", ", ")", Arrays.asList(classMatcher, akidMatcher, secretMatcher));
+        }
+    }
+
+    private static AwsCredentialsMatcher hasCredentials(String akid, String secret) {
+        return new AwsCredentialsMatcher(akid, secret);
+    }
+
+    @Test
+    public void testSingleProvider() {
+        AwsCredentialsProvider provider = decoder.decodeValue(credentialName1);
+        assertThat(provider, hasCredentials(TEST_ACCESS_KEY_ID, TEST_SECRET_KEY));
+    }
+
+    @Test
+    public void testTwoProviders() {
+        AwsCredentialsProvider provider = decoder.decodeValue(credentialName1 + "," + credentialName1);
+        assertThat(provider, hasCredentials(TEST_ACCESS_KEY_ID, TEST_SECRET_KEY));
+    }
+
+    @Test
+    public void testProfileProviderWithOneArg() {
+        AwsCredentialsProvider provider = decoder.decodeValue(credentialName2 + "|arg");
+        assertThat(provider, hasCredentials("arg", "blank"));
+    }
+
+    @Test
+    public void testProfileProviderWithTwoArgs() {
+        AwsCredentialsProvider provider = decoder.decodeValue(credentialName2 + "|arg1|arg2");
+        assertThat(provider, hasCredentials("arg1", "arg2"));
+    }
+
+    /**
+     * Test that providers in the multi-lang auth package can be resolved and instantiated.
+     */
+    @Test
+    public void testKclAuthProvider() {
+        for (final String className : Arrays.asList(
+                KclStsAssumeRoleCredentialsProvider.class.getName(), // fully-qualified name
+                KclStsAssumeRoleCredentialsProvider.class.getSimpleName(), // name-only; needs prefix
+                StsAssumeRoleCredentialsProvider.class.getName(), // user passes full sts package path
+                StsAssumeRoleCredentialsProvider.class.getSimpleName())) {
+            final AwsCredentialsProvider provider =
+                    decoder.decodeValue(className + "|arn|sessionName|endpointRegion=us-east-1");
+            assertNotNull(className, provider);
+        }
+    }
+
+    /**
+     * Test that OneArgCreateProvider in the SDK v2 can process a create() method
+     */
+    @Test
+    public void testEmptyCreateProvider() {
+        AwsCredentialsProvider provider = decoder.decodeValue(createCredentialClass);
+        assertThat(provider, hasCredentials(TEST_ACCESS_KEY_ID, TEST_SECRET_KEY));
+    }
+
+    /**
+     * Test that OneArgCreateProvider in the SDK v2 can process a create(arg1) method
+     */
+    @Test
+    public void testOneArgCreateProvider() {
+        AwsCredentialsProvider provider = decoder.decodeValue(createCredentialClass + "|testCreateProperty");
+        assertThat(provider, hasCredentials("testCreateProperty", TEST_SECRET_KEY));
+    }
+
+    /**
+     * Test that a provider can be instantiated by its varargs constructor.
+     */
+    @Test
+    public void testVarArgAuthProvider() {
+        final String[] args = new String[] {"arg1", "arg2", "arg3"};
+        final String className = VarArgCredentialsProvider.class.getName();
+        final String encodedValue = className + "|" + String.join("|", args);
+
+        final AwsCredentialsProvider provider = decoder.decodeValue(encodedValue);
+        assertEquals(Arrays.toString(args), provider.resolveCredentials().accessKeyId());
+    }
+
+    /**
+     * This credentials provider will always succeed
+     */
+    public static class AlwaysSucceedCredentialsProvider implements AwsCredentialsProvider {
+        @Override
+        public AwsCredentials resolveCredentials() {
+            return AwsBasicCredentials.create(TEST_ACCESS_KEY_ID, TEST_SECRET_KEY);
+        }
+    }
+
+    /**
+     * This credentials provider needs a constructor call to instantiate it
+     */
+    public static class ConstructorCredentialsProvider implements AwsCredentialsProvider {
+
+        private String arg1;
+        private String arg2;
+
+        @SuppressWarnings("unused")
+        public ConstructorCredentialsProvider(String arg1) {
+            this(arg1, "blank");
+        }
+
+        public ConstructorCredentialsProvider(String arg1, String arg2) {
+            this.arg1 = arg1;
+            this.arg2 = arg2;
+        }
+
+        @Override
+        public AwsCredentials resolveCredentials() {
+            return AwsBasicCredentials.create(arg1, arg2);
+        }
+    }
+
+    private static class VarArgCredentialsProvider implements AwsCredentialsProvider {
+
+        private final String[] args;
+
+        public VarArgCredentialsProvider(final String[] args) {
+            this.args = args;
+        }
+
+        @Override
+        public AwsCredentials resolveCredentials() {
+            // KISS solution to surface the constructor args
+            final String flattenedArgs = Arrays.toString(args);
+            return AwsBasicCredentials.create(flattenedArgs, flattenedArgs);
+        }
+    }
+
+    /**
+     * Credentials provider to test AWS SDK v2 create() methods for providers like ProfileCredentialsProvider
+     */
+    public static class CreateProvider implements AwsCredentialsProvider {
+        private String accessKeyId;
+
+        private CreateProvider(String accessKeyId) {
+            this.accessKeyId = accessKeyId;
+        }
+
+        public static CreateProvider create() {
+            return new CreateProvider(TEST_ACCESS_KEY_ID);
+        }
+
+        public static CreateProvider create(String accessKeyId) {
+            return new CreateProvider(accessKeyId);
+        }
+
+        @Override
+        public AwsCredentials resolveCredentials() {
+            return AwsBasicCredentials.create(accessKeyId, TEST_SECRET_KEY);
+        }
+    }
+}

amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/BuilderDynaBeanTest.java

@@ -0,0 +1,900 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.multilang.config;
+
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+
+import lombok.Builder;
+import lombok.EqualsAndHashCode;
+import lombok.RequiredArgsConstructor;
+import lombok.ToString;
+import lombok.experimental.Accessors;
+import org.apache.commons.beanutils.BeanUtilsBean;
+import org.apache.commons.beanutils.ConvertUtilsBean;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.CoreMatchers.nullValue;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.fail;
+
+public class BuilderDynaBeanTest {
+
+    private static boolean isBad = true;
+    private ConvertUtilsBean convertUtilsBean;
+    private BeanUtilsBean utilsBean;
+
+    @Rule
+    public final ExpectedException thrown = ExpectedException.none();
+
+    @Before
+    public void setup() {
+        convertUtilsBean = new ConvertUtilsBean();
+        utilsBean = new BeanUtilsBean(convertUtilsBean);
+    }
+
+    @Test
+    public void testSimpleCreateAllParameters() throws Exception {
+        BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSimpleCreate.class, convertUtilsBean);
+
+        utilsBean.setProperty(builderDynaBean, "[0]", "first");
+        utilsBean.setProperty(builderDynaBean, "[1]", "last");
+
+        TestSimpleCreate expected = TestSimpleCreate.create("first", "last");
+        TestSimpleCreate actual = builderDynaBean.build(TestSimpleCreate.class);
+
+        assertThat(actual, equalTo(expected));
+    }
+
+    @Test
+    public void testSimpleCreateToManyParameters() throws Exception {
+        thrown.expect(IllegalArgumentException.class);
+        thrown.expectMessage(containsString("exceeds the maximum"));
+
+        BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSimpleCreate.class, convertUtilsBean);
+
+        utilsBean.setProperty(builderDynaBean, "[0]", "first");
+        utilsBean.setProperty(builderDynaBean, "[1]", "last");
+        utilsBean.setProperty(builderDynaBean, "[2]", "age");
+
+        TestSimpleCreate expected = TestSimpleCreate.create("first", "last");
+        TestSimpleCreate actual = builderDynaBean.build(TestSimpleCreate.class);
+
+        assertThat(actual, equalTo(expected));
+    }
+
+    @Test
+    public void testSimpleCreateMissingParameter() throws Exception {
+        TestSimpleCreate expected = TestSimpleCreate.create(null, "last");
+
+        BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSimpleCreate.class, convertUtilsBean);
+        utilsBean.setProperty(builderDynaBean, "[1]", expected.lastName);
+
+        TestSimpleCreate actual = builderDynaBean.build(TestSimpleCreate.class);
+
+        assertThat(actual, equalTo(expected));
+    }
+
+    @Test
+    public void testSimpleCreateNoParameters() throws Exception {
+        TestSimpleCreate expected = TestSimpleCreate.create(null, null);
+
+        BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSimpleCreate.class, convertUtilsBean);
+        utilsBean.setProperty(builderDynaBean, "[1]", expected.lastName);
+
+        TestSimpleCreate actual = builderDynaBean.build(TestSimpleCreate.class);
+
+        assertThat(actual, equalTo(expected));
+    }
+
+    @Test
+    public void testComplexCreateAllParameters() throws Exception {
+        TestComplexCreate expected = TestComplexCreate.create(
+                "real", TestSimpleBuilder.builder().stringL1("l1").longVal(10L).build());
+
+        BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreate.class, convertUtilsBean);
+        utilsBean.setProperty(builderDynaBean, "[0]", expected.realName);
+        utilsBean.setProperty(builderDynaBean, "[1].stringL1", expected.test1.stringL1);
+        utilsBean.setProperty(builderDynaBean, "[1].longVal", expected.test1.longVal);
+
+        TestComplexCreate actual = builderDynaBean.build(TestComplexCreate.class);
+
+        assertThat(actual, equalTo(expected));
+    }
+
+    @Test
+    public void testComplexCreateSimpleParameterOnly() throws Exception {
+        TestComplexCreate expected = TestComplexCreate.create("real", null);
+
+        BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreate.class, convertUtilsBean);
+        utilsBean.setProperty(builderDynaBean, "[0]", expected.realName);
+
+        TestComplexCreate actual = builderDynaBean.build(TestComplexCreate.class);
+
+        assertThat(actual, equalTo(expected));
+    }
+
+    @Test
+    public void testComplexCreateComplexParameterOnly() throws Exception {
+        TestComplexCreate expected = TestComplexCreate.create(
+                null, TestSimpleBuilder.builder().stringL1("l1").longVal(10L).build());
+
+        BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreate.class, convertUtilsBean);
+        utilsBean.setProperty(builderDynaBean, "[1].stringL1", expected.test1.stringL1);
+        utilsBean.setProperty(builderDynaBean, "[1].longVal", expected.test1.longVal);
+
+        TestComplexCreate actual = builderDynaBean.build(TestComplexCreate.class);
+
+        assertThat(actual, equalTo(expected));
+    }
+
+    @Test
+    public void testComplexCreateNoParameters() throws Exception {
+        TestComplexCreate expected = TestComplexCreate.create(null, null);
+
+        BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreate.class, convertUtilsBean);
+
+        TestComplexCreate actual = builderDynaBean.build(TestComplexCreate.class);
+
+        assertThat(actual, equalTo(expected));
+    }
+
+    @Test
+    public void testSimpleBuilderAllParameters() throws Exception {
+        TestSimpleBuilder expected =
+                TestSimpleBuilder.builder().stringL1("l1").longVal(10L).build();
+
+        BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSimpleBuilder.class, convertUtilsBean);
+        utilsBean.setProperty(builderDynaBean, "stringL1", expected.stringL1);
+        utilsBean.setProperty(builderDynaBean, "longVal", expected.longVal);
+
+        TestSimpleBuilder actual = builderDynaBean.build(TestSimpleBuilder.class);
+
+        assertThat(actual, equalTo(expected));
+    }
+
+    @Test
+    public void testSimpleBuilderMissingStringL1() throws Exception {
+        TestSimpleBuilder expected = TestSimpleBuilder.builder().longVal(10L).build();
+
+        BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSimpleBuilder.class, convertUtilsBean);
+        utilsBean.setProperty(builderDynaBean, "longVal", expected.longVal);
+
+        TestSimpleBuilder actual = builderDynaBean.build(TestSimpleBuilder.class);
+
+        assertThat(actual, equalTo(expected));
+    }
+
+    @Test
+    public void testSimpleBuilderMissingLongVal() throws Exception {
+        TestSimpleBuilder expected = TestSimpleBuilder.builder().stringL1("l1").build();
+
+        BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSimpleBuilder.class, convertUtilsBean);
+        utilsBean.setProperty(builderDynaBean, "stringL1", expected.stringL1);
+
+        TestSimpleBuilder actual = builderDynaBean.build(TestSimpleBuilder.class);
+
+        assertThat(actual, equalTo(expected));
+    }
+
+    @Test
+    public void testSimpleBuilderInvalidProperty() throws Exception {
+        thrown.expect(IllegalArgumentException.class);
+        thrown.expectMessage("Unknown property: invalidProperty");
+
+        TestSimpleBuilder expected = TestSimpleBuilder.builder().build();
+
+        BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSimpleBuilder.class, convertUtilsBean);
+        utilsBean.setProperty(builderDynaBean, "invalidProperty", "invalid");
+
+        TestSimpleBuilder actual = builderDynaBean.build(TestSimpleBuilder.class);
+
+        assertThat(actual, equalTo(expected));
+    }
+
+    @Test
+    public void testComplexCreateSimpleBuilderVariantAllParameters() throws Exception {
+        TestSimpleBuilder variant =
+                TestSimpleBuilder.builder().longVal(10L).stringL1("variant").build();
+        TestComplexCreateVariance expected = TestComplexCreateVariance.create("simple-builder", variant);
+
+        BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean);
+        utilsBean.setProperty(builderDynaBean, "[0]", expected.varianceName);
+        utilsBean.setProperty(
+                builderDynaBean, "[1].class", expected.variant.getClass().getName());
+        utilsBean.setProperty(builderDynaBean, "[1].longVal", variant.longVal);
+        utilsBean.setProperty(builderDynaBean, "[1].stringL1", variant.stringL1);
+
+        TestComplexCreateVariance actual = builderDynaBean.build(TestComplexCreateVariance.class);
+
+        assertThat(actual, equalTo(expected));
+    }
+
+    @Test
+    public void testComplexCreateVariantBuilderAllParameters() throws Exception {
+        TestVariantBuilder variant = TestVariantBuilder.builder()
+                .variantBuilderName("variant-build")
+                .intClass(20)
+                .testEnum(TestEnum.Blue)
+                .build();
+        TestComplexCreateVariance expected = TestComplexCreateVariance.create("builder-variant", variant);
+
+        BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean);
+        utilsBean.setProperty(builderDynaBean, "[0]", expected.varianceName);
+        utilsBean.setProperty(builderDynaBean, "[1].class", variant.getClass().getName());
+        utilsBean.setProperty(builderDynaBean, "[1].variantBuilderName", variant.variantBuilderName);
+        utilsBean.setProperty(builderDynaBean, "[1].intClass", variant.intClass);
+        utilsBean.setProperty(builderDynaBean, "[1].testEnum", variant.testEnum);
+
+        TestComplexCreateVariance actual = builderDynaBean.build(TestComplexCreateVariance.class);
+
+        assertThat(actual, equalTo(expected));
+    }
+
+    @Test
+    public void testComplexCreateVariantCreateAllParameters() throws Exception {
+        TestVariantCreate variant = TestVariantCreate.create("variant-create", 100L, "varied");
+        TestComplexCreateVariance expected = TestComplexCreateVariance.create("create-variant", variant);
+
+        BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean);
+        utilsBean.setProperty(builderDynaBean, "[0]", expected.varianceName);
+        utilsBean.setProperty(builderDynaBean, "[1].class", variant.getClass().getName());
+        utilsBean.setProperty(builderDynaBean, "[1].[0]", variant.variantCreateName);
+        utilsBean.setProperty(builderDynaBean, "[1].[1]", variant.longClass);
+        utilsBean.setProperty(builderDynaBean, "[1].[2]", variant.varyString);
+
+        TestComplexCreateVariance actual = builderDynaBean.build(TestComplexCreateVariance.class);
+
+        assertThat(actual, equalTo(expected));
+    }
+
+    @Test
+    public void testComplexCreateVariantBuilderAllParametersPrefixWithJoiner() throws Exception {
+        TestVariantBuilder variant = TestVariantBuilder.builder()
+                .variantBuilderName("variant-build")
+                .intClass(20)
+                .testEnum(TestEnum.Blue)
+                .build();
+        TestComplexCreateVariance expected = TestComplexCreateVariance.create("builder-variant-prefix", variant);
+
+        String prefix = variant.getClass().getEnclosingClass().getName() + "$";
+        BuilderDynaBean builderDynaBean =
+                new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean, prefix);
+        utilsBean.setProperty(builderDynaBean, "[0]", expected.varianceName);
+        utilsBean.setProperty(builderDynaBean, "[1].class", variant.getClass().getSimpleName());
+        utilsBean.setProperty(builderDynaBean, "[1].variantBuilderName", variant.variantBuilderName);
+        utilsBean.setProperty(builderDynaBean, "[1].intClass", variant.intClass);
+        utilsBean.setProperty(builderDynaBean, "[1].testEnum", variant.testEnum);
+
+        TestComplexCreateVariance actual = builderDynaBean.build(TestComplexCreateVariance.class);
+
+        assertThat(actual, equalTo(expected));
+    }
+
+    @Test
+    public void testComplexCreateVariantBuilderAllParametersPrefixWithOutJoiner() throws Exception {
+        TestVariantBuilder variant = TestVariantBuilder.builder()
+                .variantBuilderName("variant-build")
+                .intClass(20)
+                .testEnum(TestEnum.Blue)
+                .build();
+        TestComplexCreateVariance expected = TestComplexCreateVariance.create("builder-variant-prefix", variant);
+
+        String prefix = variant.getClass().getEnclosingClass().getName();
+        BuilderDynaBean builderDynaBean =
+                new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean, prefix);
+        utilsBean.setProperty(builderDynaBean, "[0]", expected.varianceName);
+        utilsBean.setProperty(builderDynaBean, "[1].class", variant.getClass().getSimpleName());
+        utilsBean.setProperty(builderDynaBean, "[1].variantBuilderName", variant.variantBuilderName);
+        utilsBean.setProperty(builderDynaBean, "[1].intClass", variant.intClass);
+        utilsBean.setProperty(builderDynaBean, "[1].testEnum", variant.testEnum);
+
+        TestComplexCreateVariance actual = builderDynaBean.build(TestComplexCreateVariance.class);
+
+        assertThat(actual, equalTo(expected));
+    }
+
+    @Test
+    public void testComplexCreateVariantInvalidVariantClass() throws Exception {
+        String invalidClass = "invalid-class";
+        thrown.expect(IllegalArgumentException.class);
+        thrown.expectMessage(containsString("Unable to load class"));
+        thrown.expectMessage(containsString(invalidClass));
+        thrown.expectMessage(containsString("Attempted"));
+
+        TestComplexCreateVariance expected = TestComplexCreateVariance.create("builder-variant-prefix", null);
+
+        BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean);
+        utilsBean.setProperty(builderDynaBean, "[0]", expected.varianceName);
+        utilsBean.setProperty(builderDynaBean, "[1].class", invalidClass);
+    }
+
+    @Test
+    public void testComplexCreateVariantBadLoadClass() throws Exception {
+        thrown.expect(ExceptionInInitializerError.class);
+        thrown.expectCause(instanceOf(BadClassException.class));
+        TestComplexCreateVariance expected = TestComplexCreateVariance.create("builder-variant-prefix", null);
+
+        BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean);
+        utilsBean.setProperty(builderDynaBean, "[0]", expected.varianceName);
+        utilsBean.setProperty(builderDynaBean, "[1].class", getClass().getName() + "$BadClass");
+    }
+
+    @Test
+    public void testComplexRootAllParameters() throws Exception {
+        TestSimpleBuilder simpleBuilder =
+                TestSimpleBuilder.builder().stringL1("simple-l1").longVal(20L).build();
+        TestRootClass expected = TestRootClass.builder()
+                .intVal(10)
+                .stringVal("root")
+                .testEnum(TestEnum.Red)
+                .testComplexCreate(TestComplexCreate.create(
+                        "real",
+                        TestSimpleBuilder.builder()
+                                .stringL1("complex-l1")
+                                .longVal(10L)
+                                .build()))
+                .testSimpleBuilder(simpleBuilder)
+                .testSimpleCreate(TestSimpleCreate.create("first", "last"))
+                .build();
+
+        BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestRootClass.class, convertUtilsBean);
+
+        utilsBean.setProperty(builderDynaBean, "intVal", expected.intVal);
+        utilsBean.setProperty(builderDynaBean, "stringVal", expected.stringVal);
+        utilsBean.setProperty(builderDynaBean, "testEnum", expected.testEnum);
+        utilsBean.setProperty(builderDynaBean, "testComplexCreate.[0]", expected.testComplexCreate.realName);
+        utilsBean.setProperty(
+                builderDynaBean, "testComplexCreate.[1].stringL1", expected.testComplexCreate.test1.stringL1);
+        utilsBean.setProperty(
+                builderDynaBean, "testComplexCreate.[1].longVal", expected.testComplexCreate.test1.longVal);
+        utilsBean.setProperty(builderDynaBean, "testSimpleBuilder.class", TestSimpleBuilder.class.getName());
+        utilsBean.setProperty(builderDynaBean, "testSimpleBuilder.stringL1", simpleBuilder.stringL1);
+        utilsBean.setProperty(builderDynaBean, "testSimpleBuilder.longVal", simpleBuilder.longVal);
+        utilsBean.setProperty(builderDynaBean, "testSimpleCreate.[0]", expected.testSimpleCreate.firstName);
+        utilsBean.setProperty(builderDynaBean, "testSimpleCreate.[1]", expected.testSimpleCreate.lastName);
+
+        TestRootClass actual = builderDynaBean.build(TestRootClass.class);
+
+        assertThat(actual, equalTo(expected));
+    }
+
+    @Test
+    public void testComplexRootNoParameters() throws Exception {
+        TestRootClass expected = TestRootClass.builder().build();
+
+        BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestRootClass.class, convertUtilsBean);
+
+        TestRootClass actual = builderDynaBean.build(TestRootClass.class);
+
+        assertThat(actual, equalTo(expected));
+    }
+
+    @Test
+    public void testComplexRootTopLevelOnly() throws Exception {
+        TestRootClass expected = TestRootClass.builder()
+                .intVal(10)
+                .stringVal("root")
+                .testEnum(TestEnum.Red)
+                .build();
+
+        BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestRootClass.class, convertUtilsBean);
+
+        utilsBean.setProperty(builderDynaBean, "intVal", expected.intVal);
+        utilsBean.setProperty(builderDynaBean, "stringVal", expected.stringVal);
+        utilsBean.setProperty(builderDynaBean, "testEnum", expected.testEnum);
+
+        TestRootClass actual = builderDynaBean.build(TestRootClass.class);
+
+        assertThat(actual, equalTo(expected));
+    }
+
+    @Test
+    public void testSupplierNotUsed() throws Exception {
+        TestVariantBuilder variant = TestVariantBuilder.builder()
+                .testEnum(TestEnum.Green)
+                .intClass(10)
+                .variantBuilderName("variant-supplier")
+                .build();
+        TestSupplierClass expected =
+                TestSupplierClass.builder().variantClass(variant).build();
+
+        BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSupplierClass.class, convertUtilsBean);
+        utilsBean.setProperty(
+                builderDynaBean, "variantClass.class", variant.getClass().getName());
+        utilsBean.setProperty(builderDynaBean, "variantClass.testEnum", variant.testEnum);
+        utilsBean.setProperty(builderDynaBean, "variantClass.intClass", variant.intClass);
+        utilsBean.setProperty(builderDynaBean, "variantClass.variantBuilderName", variant.variantBuilderName);
+
+        TestSupplierClass actual = builderDynaBean.build(TestSupplierClass.class);
+
+        assertThat(actual, equalTo(expected));
+    }
+
+    @Test
+    public void testConsumerMethodsNotExposed() throws Exception {
+        thrown.expect(IllegalArgumentException.class);
+        thrown.expectMessage(containsString("Unknown property: mutator"));
+
+        BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSupplierClass.class, convertUtilsBean);
+        utilsBean.setProperty(builderDynaBean, "mutator", "test-value");
+    }
+
+    @Test
+    public void testAttemptToBuildForWrongClass() throws Exception {
+        thrown.expect(IllegalArgumentException.class);
+        thrown.expectMessage(containsString("cannot be assigned to"));
+        thrown.expectMessage(containsString(TestVariantCreate.class.getName()));
+        thrown.expectMessage(containsString(TestVariantBuilder.class.getName()));
+
+        BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestVariantBuilder.class, convertUtilsBean);
+        builderDynaBean.build(TestVariantCreate.class);
+    }
+
+    @Test
+    public void testVariantBuildsToSuperType() throws Exception {
+        TestVariantBuilder expected = TestVariantBuilder.builder()
+                .intClass(10)
+                .testEnum(TestEnum.Green)
+                .variantBuilderName("variant-super")
+                .build();
+
+        BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestInterface.class, convertUtilsBean);
+        utilsBean.setProperty(builderDynaBean, "class", expected.getClass().getName());
+        utilsBean.setProperty(builderDynaBean, "intClass", expected.intClass);
+        utilsBean.setProperty(builderDynaBean, "testEnum", expected.testEnum);
+        utilsBean.setProperty(builderDynaBean, "variantBuilderName", expected.variantBuilderName);
+
+        TestInterface actual = builderDynaBean.build(TestInterface.class);
+
+        assertThat(actual, equalTo(expected));
+    }
+
+    @Test
+    public void testEmptyPropertyHandler() throws Exception {
+        String emptyPropertyValue = "test-property";
+        TestVariantCreate expected = TestVariantCreate.create(
+                emptyPropertyValue, (long) emptyPropertyValue.length(), emptyPropertyValue + "-vary");
+        BuilderDynaBean builderDynaBean = new BuilderDynaBean(
+                TestInterface.class,
+                convertUtilsBean,
+                s -> TestVariantCreate.create(s, (long) s.length(), s + "-vary"));
+        utilsBean.setProperty(builderDynaBean, "", emptyPropertyValue);
+
+        TestInterface actual = builderDynaBean.build(TestInterface.class);
+
+        assertThat(actual, equalTo(expected));
+    }
+
+    @Test
+    public void testEmptyPropertyHandlerThrowsAfterUse() throws Exception {
+        thrown.expect(IllegalStateException.class);
+        thrown.expectMessage(containsString("When a property handler is resolved further properties may not be set."));
+
+        BuilderDynaBean builderDynaBean = new BuilderDynaBean(
+                TestInterface.class, convertUtilsBean, s -> TestVariantCreate.create("test", 10, "test"));
+        utilsBean.setProperty(builderDynaBean, "", "test");
+        utilsBean.setProperty(builderDynaBean, "[0]", "test");
+    }
+
+    @Test
+    public void testEmptyPropertyReturnsInvalidObject() throws Exception {
+        thrown.expect(IllegalArgumentException.class);
+        thrown.expectMessage(containsString(TestEnum.class.getName()));
+        thrown.expectMessage(containsString(TestInterface.class.getName()));
+        thrown.expectMessage(containsString("cannot be assigned to"));
+
+        BuilderDynaBean builderDynaBean =
+                new BuilderDynaBean(TestInterface.class, convertUtilsBean, s -> TestEnum.Green);
+
+        utilsBean.setProperty(builderDynaBean, "", "test");
+
+        builderDynaBean.build(TestInterface.class);
+    }
+
+    @Test
+    public void testSimpleArrayValues() throws Exception {
+        SimpleArrayClassVariant expected = SimpleArrayClassVariant.builder()
+                .ints(new Integer[] {1, 2, 3})
+                .variantName("simple-array")
+                .longs(new Long[] {1L, 2L, 3L})
+                .strings(new String[] {"a", "b", "c"})
+                .build();
+
+        BuilderDynaBean builderDynaBean = new BuilderDynaBean(SimpleArrayClassVariant.class, convertUtilsBean);
+        utilsBean.setProperty(builderDynaBean, "variantName", expected.variantName);
+        for (int i = 0; i < expected.strings.length; ++i) {
+            utilsBean.setProperty(builderDynaBean, "strings[" + i + "]", expected.strings[i]);
+        }
+
+        for (int i = 0; i < expected.ints.length; ++i) {
+            utilsBean.setProperty(builderDynaBean, "ints[" + i + "]", expected.ints[i]);
+        }
+
+        for (int i = 0; i < expected.longs.length; ++i) {
+            utilsBean.setProperty(builderDynaBean, "longs[" + i + "]", expected.longs[i]);
+        }
+
+        SimpleArrayClassVariant actual = builderDynaBean.build(SimpleArrayClassVariant.class);
+
+        assertThat(actual, equalTo(expected));
+    }
+
+    @Test
+    public void testComplexArrayValuesBuilder() throws Exception {
+        TestVariantBuilder variant1 = TestVariantBuilder.builder()
+                .variantBuilderName("variant-1")
+                .testEnum(TestEnum.Green)
+                .intClass(10)
+                .build();
+        TestVariantBuilder variant2 = TestVariantBuilder.builder()
+                .variantBuilderName("variant-2")
+                .testEnum(TestEnum.Blue)
+                .intClass(20)
+                .build();
+        ComplexArrayClassVariant expected = ComplexArrayClassVariant.builder()
+                .variantName("complex-test")
+                .tests(new TestInterface[] {variant1, variant2})
+                .build();
+
+        BuilderDynaBean builderDynaBean = new BuilderDynaBean(ComplexArrayClassVariant.class, convertUtilsBean);
+
+        utilsBean.setProperty(builderDynaBean, "variantName", expected.variantName);
+        utilsBean.setProperty(builderDynaBean, "tests[0].class", TestVariantBuilder.class.getName());
+        utilsBean.setProperty(builderDynaBean, "tests[0].variantBuilderName", variant1.variantBuilderName);
+        utilsBean.setProperty(builderDynaBean, "tests[0].intClass", variant1.intClass);
+        utilsBean.setProperty(builderDynaBean, "tests[0].testEnum", variant1.testEnum);
+
+        utilsBean.setProperty(builderDynaBean, "tests[1].class", TestVariantBuilder.class.getName());
+        utilsBean.setProperty(builderDynaBean, "tests[1].variantBuilderName", variant2.variantBuilderName);
+        utilsBean.setProperty(builderDynaBean, "tests[1].intClass", variant2.intClass);
+        utilsBean.setProperty(builderDynaBean, "tests[1].testEnum", variant2.testEnum);
+
+        ComplexArrayClassVariant actual = builderDynaBean.build(ComplexArrayClassVariant.class);
+
+        assertThat(actual, equalTo(expected));
+    }
+
+    @Test
+    public void testComplexArrayValuesCreate() throws Exception {
+        TestVariantCreate variant1 = TestVariantCreate.create("variant-1", 10L, "vary-1");
+        TestVariantCreate variant2 = TestVariantCreate.create("variant-2", 20L, "vary-2");
+
+        ComplexArrayClassVariant expected = ComplexArrayClassVariant.builder()
+                .variantName("create-test")
+                .tests(new TestInterface[] {variant1, variant2})
+                .build();
+
+        BuilderDynaBean builderDynaBean = new BuilderDynaBean(ComplexArrayClassVariant.class, convertUtilsBean);
+
+        utilsBean.setProperty(builderDynaBean, "variantName", expected.variantName);
+        utilsBean.setProperty(
+                builderDynaBean, "tests[0].class", variant1.getClass().getName());
+        utilsBean.setProperty(builderDynaBean, "tests[0].[0]", variant1.variantCreateName);
+        utilsBean.setProperty(builderDynaBean, "tests[0].[1]", variant1.longClass);
+        utilsBean.setProperty(builderDynaBean, "tests[0].[2]", variant1.varyString);
+
+        utilsBean.setProperty(
+                builderDynaBean, "tests[1].class", variant2.getClass().getName());
+        utilsBean.setProperty(builderDynaBean, "tests[1].[0]", variant2.variantCreateName);
+        utilsBean.setProperty(builderDynaBean, "tests[1].[1]", variant2.longClass);
+        utilsBean.setProperty(builderDynaBean, "tests[1].[2]", variant2.varyString);
+
+        ComplexArrayClassVariant actual = builderDynaBean.build(ComplexArrayClassVariant.class);
+
+        assertThat(actual, equalTo(expected));
+    }
+
+    @Test
+    public void testComplexArrayValuesMixed() throws Exception {
+        TestInterface[] variants = new TestInterface[10];
+        for (int i = 0; i < variants.length; ++i) {
+            if (i % 2 == 0) {
+                variants[i] = TestVariantCreate.create("create-variant-" + i, i + 5, "vary-" + i);
+            } else {
+                variants[i] = TestVariantBuilder.builder()
+                        .testEnum(TestEnum.values()[i % TestEnum.values().length])
+                        .intClass(i)
+                        .variantBuilderName("builder-variant-" + i)
+                        .build();
+            }
+        }
+
+        ComplexArrayClassVariant expected = ComplexArrayClassVariant.builder()
+                .variantName("large-complex")
+                .tests(variants)
+                .build();
+
+        BuilderDynaBean builderDynaBean = new BuilderDynaBean(ComplexArrayClassVariant.class, convertUtilsBean);
+
+        utilsBean.setProperty(builderDynaBean, "variantName", expected.variantName);
+        for (int i = 0; i < variants.length; ++i) {
+            String prefix = "tests[" + i + "].";
+            TestInterface variant = variants[i];
+            if (variant instanceof TestVariantCreate) {
+                TestVariantCreate create = (TestVariantCreate) variant;
+                utilsBean.setProperty(
+                        builderDynaBean, prefix + "class", create.getClass().getName());
+                utilsBean.setProperty(builderDynaBean, prefix + "[0]", create.variantCreateName);
+                utilsBean.setProperty(builderDynaBean, prefix + "[1]", create.longClass);
+                utilsBean.setProperty(builderDynaBean, prefix + "[2]", create.varyString);
+            } else if (variant instanceof TestVariantBuilder) {
+                TestVariantBuilder builder = (TestVariantBuilder) variant;
+                utilsBean.setProperty(
+                        builderDynaBean, prefix + "class", builder.getClass().getName());
+                utilsBean.setProperty(builderDynaBean, prefix + "variantBuilderName", builder.variantBuilderName);
+                utilsBean.setProperty(builderDynaBean, prefix + "intClass", builder.intClass);
+                utilsBean.setProperty(builderDynaBean, prefix + "testEnum", builder.testEnum);
+            } else {
+                fail("Unknown variant " + variants[i].getClass().getName());
+            }
+        }
+
+        ComplexArrayClassVariant actual = builderDynaBean.build(ComplexArrayClassVariant.class);
+
+        assertThat(actual, equalTo(expected));
+    }
+
+    @Test
+    public void testInvalidBuilderCreateClassBuild() throws Exception {
+        BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestInterface.class, convertUtilsBean);
+
+        TestInterface actual = builderDynaBean.build(TestInterface.class);
+
+        assertThat(actual, nullValue());
+    }
+
+    @Test
+    public void testInvalidBuilderCreateClassSetProperty() throws Exception {
+        thrown.expect(IllegalStateException.class);
+        thrown.expectMessage(containsString("Unable to to introspect or handle"));
+        thrown.expectMessage(containsString(TestInterface.class.getName()));
+        thrown.expectMessage(containsString("as it doesn't have a builder or create method"));
+
+        BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestInterface.class, convertUtilsBean);
+
+        utilsBean.setProperty(builderDynaBean, "testProperty", "test");
+    }
+
+    @Test
+    public void testSetMapAccessThrowsException() throws Exception {
+        thrown.expect(UnsupportedOperationException.class);
+        thrown.expectMessage(BuilderDynaBean.NO_MAP_ACCESS_SUPPORT);
+
+        BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSimpleBuilder.class, convertUtilsBean);
+
+        utilsBean.setProperty(builderDynaBean, "stringL1(value)", "test");
+    }
+
+    @Test
+    public void testGetMapAccessThrowsException() throws Exception {
+        thrown.expect(UnsupportedOperationException.class);
+        thrown.expectMessage(BuilderDynaBean.NO_MAP_ACCESS_SUPPORT);
+
+        BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSimpleBuilder.class, convertUtilsBean);
+        //
+        // We directly access the get method as there is no way to trigger utilsBean to access it
+        //
+        builderDynaBean.get("stringL1", "value");
+    }
+
+    @Test
+    public void testRemoveThrowsException() throws Exception {
+        thrown.expect(UnsupportedOperationException.class);
+        thrown.expectMessage(BuilderDynaBean.NO_MAP_ACCESS_SUPPORT);
+
+        BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSimpleBuilder.class, convertUtilsBean);
+        //
+        // We directly access the remove method as there is no way to trigger utilsBean to access it
+        //
+        builderDynaBean.remove("stringL1", "value");
+    }
+
+    @Test
+    public void testContainsThrowsException() throws Exception {
+        thrown.expect(UnsupportedOperationException.class);
+        thrown.expectMessage(BuilderDynaBean.NO_MAP_ACCESS_SUPPORT);
+
+        BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSimpleBuilder.class, convertUtilsBean);
+        //
+        // We directly access the remove method as there is no way to trigger utilsBean to access it
+        //
+        builderDynaBean.contains("stringL1", "value");
+    }
+
+    @Test
+    public void testAdditionalMutators() throws Exception {
+        TestSimpleBuilder expected =
+                TestSimpleBuilder.builder().stringL1("test").longVal(10L).build();
+
+        BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSimpleBuilder.class, convertUtilsBean);
+
+        utilsBean.setProperty(builderDynaBean, "stringL1", expected.stringL1);
+
+        TestSimpleBuilder actual =
+                builderDynaBean.build(TestSimpleBuilder.class, b -> ((TestSimpleBuilder.TestSimpleBuilderBuilder) b)
+                        .longVal(expected.longVal));
+
+        assertThat(actual, equalTo(expected));
+    }
+
+    public enum TestEnum {
+        Red,
+        Green,
+        Blue
+    }
+
+    public interface TestInterface {}
+
+    @Accessors(fluent = true)
+    @ToString
+    @EqualsAndHashCode
+    @RequiredArgsConstructor
+    public static class TestSimpleCreate {
+        private final String firstName;
+        private final String lastName;
+
+        public static TestSimpleCreate create(String firstName, String lastName) {
+            return new TestSimpleCreate(firstName, lastName);
+        }
+    }
+
+    @Accessors(fluent = true)
+    @ToString
+    @EqualsAndHashCode
+    @RequiredArgsConstructor
+    public static class TestComplexCreate {
+        private final String realName;
+        private final TestSimpleBuilder test1;
+
+        public static TestComplexCreate create(String realName, TestSimpleBuilder test1) {
+            return new TestComplexCreate(realName, test1);
+        }
+    }
+
+    @Accessors(fluent = true)
+    @ToString
+    @EqualsAndHashCode
+    @RequiredArgsConstructor
+    public static class TestComplexCreateVariance {
+        private final String varianceName;
+        private final TestInterface variant;
+
+        public static TestComplexCreateVariance create(String varianceName, TestInterface variant) {
+            return new TestComplexCreateVariance(varianceName, variant);
+        }
+    }
+
+    @Builder
+    @Accessors(fluent = true)
+    @ToString
+    @EqualsAndHashCode
+    public static class TestSimpleBuilder implements TestInterface {
+        private String stringL1;
+        private long longVal;
+    }
+
+    @Builder
+    @Accessors(fluent = true)
+    @ToString
+    @EqualsAndHashCode
+    public static class TestVariantBuilder implements TestInterface {
+        private String variantBuilderName;
+        private TestEnum testEnum;
+        private Integer intClass;
+    }
+
+    @Accessors(fluent = true)
+    @ToString
+    @EqualsAndHashCode
+    @RequiredArgsConstructor
+    public static class TestVariantCreate implements TestInterface {
+        private final String variantCreateName;
+        private final long longClass;
+        private final String varyString;
+
+        public static TestVariantCreate create(String variantCreateName, long longClass, String varyString) {
+            return new TestVariantCreate(variantCreateName, longClass, varyString);
+        }
+    }
+
+    @Builder
+    @Accessors(fluent = true)
+    @ToString
+    @EqualsAndHashCode
+    public static class TestRootClass {
+        private String stringVal;
+        private int intVal;
+        private TestEnum testEnum;
+        TestSimpleCreate testSimpleCreate;
+        TestComplexCreate testComplexCreate;
+        TestSimpleBuilder testSimpleBuilder;
+    }
+
+    @ToString
+    @EqualsAndHashCode
+    public static class TestSupplierClass {
+        private TestInterface variantClass;
+
+        public static TestSupplierClassBuilder builder() {
+            return new TestSupplierClassBuilder();
+        }
+    }
+
+    @Builder
+    @Accessors(fluent = true)
+    @ToString
+    @EqualsAndHashCode
+    public static class SimpleArrayClassVariant implements TestInterface {
+        private String variantName;
+        private String[] strings;
+        private Integer[] ints;
+        private Long[] longs;
+    }
+
+    @Builder
+    @Accessors(fluent = true)
+    @ToString
+    @EqualsAndHashCode
+    public static class ComplexArrayClassVariant implements TestInterface {
+        private String variantName;
+        private TestInterface[] tests;
+    }
+
+    public static class TestSupplierClassBuilder {
+        private TestSupplierClass testSupplierClass = new TestSupplierClass();
+
+        public TestSupplierClassBuilder variantClass(TestInterface testInterface) {
+            testSupplierClass.variantClass = testInterface;
+            return this;
+        }
+
+        public TestSupplierClassBuilder variantClass(Supplier<TestInterface> supplier) {
+            throw new IllegalStateException("Supplier method should not be used.");
+        }
+
+        public TestSupplierClassBuilder mutator(Consumer<TestSupplierClassBuilder> consumer) {
+            consumer.accept(this);
+            return this;
+        }
+
+        public TestSupplierClass build() {
+            return testSupplierClass;
+        }
+    }
+
+    public static class BadClassException extends RuntimeException {
+        public BadClassException(String message) {
+            super(message);
+        }
+    }
+
+    public static class BadClass {
+        static {
+            if (BuilderDynaBeanTest.isBad) {
+                throw new BadClassException("This is a bad class");
+            }
+        }
+
+        public String name = "default";
+    }
+}

amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/ConfigurationSettableUtilsTest.java

@@ -0,0 +1,234 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.multilang.config;
+
+import java.util.Optional;
+
+import lombok.Builder;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.experimental.Accessors;
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.junit.Assert.assertThat;
+
+public class ConfigurationSettableUtilsTest {
+
+    @Test
+    public void testNoPropertiesSet() {
+        ConfigResult expected = ConfigResult.builder().build();
+
+        ConfigObject configObject = ConfigObject.builder().build();
+        ConfigResult actual = resolve(configObject);
+
+        assertThat(actual, equalTo(expected));
+    }
+
+    @Test
+    public void testPrimitivesSet() {
+        ConfigResult expected = ConfigResult.builder().rawInt(10).rawLong(15L).build();
+
+        ConfigObject configObject = ConfigObject.builder()
+                .rawInt(expected.rawInt)
+                .rawLong(expected.rawLong)
+                .build();
+        ConfigResult actual = resolve(configObject);
+
+        assertThat(actual, equalTo(expected));
+    }
+
+    @Test
+    public void testBoolean() {
+        ConfigResult expected = ConfigResult.builder().bool(false).build();
+
+        ConfigObject configObject = ConfigObject.builder().bool(expected.bool).build();
+        ConfigResult actual = resolve(configObject);
+
+        assertThat(actual, equalTo(expected));
+    }
+
+    @Test
+    public void testHeapValuesSet() {
+        ConfigResult expected =
+                ConfigResult.builder().name("test").boxedInt(10).boxedLong(15L).build();
+
+        ConfigObject configObject = ConfigObject.builder()
+                .name(expected.name)
+                .boxedInt(expected.boxedInt.intValue())
+                .boxedLong(expected.boxedLong.longValue())
+                .build();
+        ConfigResult actual = resolve(configObject);
+
+        assertThat(actual, equalTo(expected));
+    }
+
+    @Test
+    public void testComplexValuesSet() {
+        ComplexValue complexValue =
+                ComplexValue.builder().name("complex").value(10).build();
+        ConfigResult expected =
+                ConfigResult.builder().complexValue(complexValue).build();
+
+        ConfigObject configObject = ConfigObject.builder()
+                .complexValue(ComplexValue.builder()
+                        .name(complexValue.name)
+                        .value(complexValue.value)
+                        .build())
+                .build();
+        ConfigResult actual = resolve(configObject);
+
+        assertThat(actual, equalTo(expected));
+    }
+
+    @Test
+    public void testOptionalValuesSet() {
+        ComplexValue complexValue =
+                ComplexValue.builder().name("optional-complex").value(20).build();
+        ConfigResult expected = ConfigResult.builder()
+                .optionalString(Optional.of("test"))
+                .optionalInteger(Optional.of(10))
+                .optionalLong(Optional.of(15L))
+                .optionalComplexValue(Optional.of(complexValue))
+                .build();
+
+        ConfigObject configObject = ConfigObject.builder()
+                .optionalString(expected.optionalString.get())
+                .optionalInteger(expected.optionalInteger.get())
+                .optionalLong(expected.optionalLong.get())
+                .optionalComplexValue(expected.optionalComplexValue.get())
+                .build();
+        ConfigResult actual = resolve(configObject);
+
+        assertThat(actual, equalTo(expected));
+    }
+
+    @Test
+    public void testRenamedRawValues() {
+        ComplexValue complexValue =
+                ComplexValue.builder().name("renamed-complex").value(20).build();
+        ConfigResult expected = ConfigResult.builder()
+                .renamedString("renamed")
+                .renamedInt(10)
+                .renamedOptionalString(Optional.of("renamed-optional"))
+                .renamedComplexValue(complexValue)
+                .build();
+
+        ConfigObject configObject = ConfigObject.builder()
+                .toRenameString(expected.renamedString)
+                .toRenameInt(expected.renamedInt)
+                .toRenameComplexValue(complexValue)
+                .optionalToRename(expected.renamedOptionalString.get())
+                .build();
+        ConfigResult actual = resolve(configObject);
+
+        assertThat(actual, equalTo(expected));
+    }
+
+    private ConfigResult resolve(ConfigObject configObject) {
+        return ConfigurationSettableUtils.resolveFields(
+                configObject, ConfigResult.builder().build());
+    }
+
+    @Accessors(fluent = true)
+    @Builder
+    @Getter
+    @Setter
+    @EqualsAndHashCode
+    public static class ConfigResult {
+        private String name;
+        private int rawInt;
+        private Integer boxedInt;
+        private long rawLong;
+        private Long boxedLong;
+        private ComplexValue complexValue;
+
+        @Builder.Default
+        private Boolean bool = true;
+
+        private Optional<String> optionalString;
+        private Optional<Integer> optionalInteger;
+        private Optional<Long> optionalLong;
+        private Optional<ComplexValue> optionalComplexValue;
+
+        private String renamedString;
+        private int renamedInt;
+        private Optional<String> renamedOptionalString;
+        private ComplexValue renamedComplexValue;
+    }
+
+    @Accessors(fluent = true)
+    @Builder
+    @EqualsAndHashCode
+    public static class ComplexValue {
+        private String name;
+        private int value;
+    }
+
+    @Builder
+    public static class ConfigObject {
+
+        @ConfigurationSettable(configurationClass = ConfigResult.class)
+        private String name;
+
+        @ConfigurationSettable(configurationClass = ConfigResult.class)
+        private int rawInt;
+
+        @ConfigurationSettable(configurationClass = ConfigResult.class)
+        @Builder.Default
+        private Boolean bool = true;
+
+        @ConfigurationSettable(configurationClass = ConfigResult.class)
+        private Integer boxedInt;
+
+        @ConfigurationSettable(configurationClass = ConfigResult.class)
+        private long rawLong;
+
+        @ConfigurationSettable(configurationClass = ConfigResult.class)
+        private Long boxedLong;
+
+        @ConfigurationSettable(configurationClass = ConfigResult.class)
+        private ComplexValue complexValue;
+
+        @ConfigurationSettable(configurationClass = ConfigResult.class, convertToOptional = true)
+        private String optionalString;
+
+        @ConfigurationSettable(configurationClass = ConfigResult.class, convertToOptional = true)
+        private Integer optionalInteger;
+
+        @ConfigurationSettable(configurationClass = ConfigResult.class, convertToOptional = true)
+        private Long optionalLong;
+
+        @ConfigurationSettable(configurationClass = ConfigResult.class, convertToOptional = true)
+        private ComplexValue optionalComplexValue;
+
+        @ConfigurationSettable(configurationClass = ConfigResult.class, methodName = "renamedString")
+        private String toRenameString;
+
+        @ConfigurationSettable(configurationClass = ConfigResult.class, methodName = "renamedInt")
+        private int toRenameInt;
+
+        @ConfigurationSettable(
+                configurationClass = ConfigResult.class,
+                methodName = "renamedOptionalString",
+                convertToOptional = true)
+        private String optionalToRename;
+
+        @ConfigurationSettable(configurationClass = ConfigResult.class, methodName = "renamedComplexValue")
+        private ComplexValue toRenameComplexValue;
+    }
+}

amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/DatePropertyValueDecoderTest.java

@@ -1,27 +1,24 @@
 /*
- * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * Licensed under the Amazon Software License (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
- * http://aws.amazon.com/asl/
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
-package com.amazonaws.services.kinesis.clientlibrary.config;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
+package software.amazon.kinesis.multilang.config;
 
 import java.util.Date;
 
 import org.junit.Test;
 
-import com.amazonaws.services.kinesis.clientlibrary.config.DatePropertyValueDecoder;
+import static org.junit.Assert.assertEquals;
 
 public class DatePropertyValueDecoderTest {
 

amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/FanoutConfigBeanTest.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.multilang.config;
+
+import org.apache.commons.beanutils.BeanUtilsBean;
+import org.apache.commons.beanutils.ConvertUtilsBean;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.runners.MockitoJUnitRunner;
+import software.amazon.awssdk.services.kinesis.KinesisAsyncClient;
+import software.amazon.kinesis.retrieval.fanout.FanOutConfig;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.junit.Assert.assertThat;
+
+@RunWith(MockitoJUnitRunner.class)
+public class FanoutConfigBeanTest {
+
+    @Mock
+    private KinesisAsyncClient kinesisAsyncClient;
+
+    @Test
+    public void testAllConfigurationTransits() {
+        FanoutConfigBean fanoutConfigBean = new FanoutConfigBean();
+        fanoutConfigBean.setConsumerArn("consumer-arn");
+        fanoutConfigBean.setConsumerName("consumer-name");
+        fanoutConfigBean.setMaxDescribeStreamConsumerRetries(10);
+        fanoutConfigBean.setMaxDescribeStreamSummaryRetries(20);
+        fanoutConfigBean.setRegisterStreamConsumerRetries(30);
+        fanoutConfigBean.setRetryBackoffMillis(1000);
+
+        ConvertUtilsBean convertUtilsBean = new ConvertUtilsBean();
+        BeanUtilsBean utilsBean = new BeanUtilsBean(convertUtilsBean);
+
+        MultiLangDaemonConfiguration configuration = new MultiLangDaemonConfiguration(utilsBean, convertUtilsBean);
+        configuration.setStreamName("test-stream");
+        configuration.setApplicationName("test-application");
+        FanOutConfig fanOutConfig = fanoutConfigBean.build(kinesisAsyncClient, configuration);
+
+        assertThat(fanOutConfig.kinesisClient(), equalTo(kinesisAsyncClient));
+        assertThat(fanOutConfig.streamName(), equalTo(configuration.getStreamName()));
+        assertThat(fanOutConfig.applicationName(), equalTo(configuration.getApplicationName()));
+        assertThat(fanOutConfig.consumerArn(), equalTo(fanoutConfigBean.getConsumerArn()));
+        assertThat(fanOutConfig.consumerName(), equalTo(fanoutConfigBean.getConsumerName()));
+        assertThat(
+                fanOutConfig.maxDescribeStreamConsumerRetries(),
+                equalTo(fanoutConfigBean.getMaxDescribeStreamConsumerRetries()));
+        assertThat(
+                fanOutConfig.maxDescribeStreamSummaryRetries(),
+                equalTo(fanoutConfigBean.getMaxDescribeStreamSummaryRetries()));
+        assertThat(
+                fanOutConfig.registerStreamConsumerRetries(),
+                equalTo(fanoutConfigBean.getRegisterStreamConsumerRetries()));
+        assertThat(fanOutConfig.retryBackoffMillis(), equalTo(fanoutConfigBean.getRetryBackoffMillis()));
+    }
+}

amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/KinesisClientLibConfiguratorTest.java

@@ -0,0 +1,759 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package software.amazon.kinesis.multilang.config;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.NoSuchElementException;
+import java.util.Set;
+
+import com.google.common.collect.ImmutableSet;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.runners.MockitoJUnitRunner;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.services.dynamodb.model.BillingMode;
+import software.amazon.kinesis.common.InitialPositionInStream;
+import software.amazon.kinesis.coordinator.CoordinatorConfig;
+import software.amazon.kinesis.metrics.MetricsLevel;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.nullValue;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+@RunWith(MockitoJUnitRunner.class)
+public class KinesisClientLibConfiguratorTest {
+
+    private final String credentialName1 = AlwaysSucceedCredentialsProvider.class.getName();
+    private final String credentialName2 = AlwaysFailCredentialsProvider.class.getName();
+    private final String credentialNameKinesis = AlwaysSucceedCredentialsProviderKinesis.class.getName();
+    private final String credentialNameDynamoDB = AlwaysSucceedCredentialsProviderDynamoDB.class.getName();
+    private final String credentialNameCloudWatch = AlwaysSucceedCredentialsProviderCloudWatch.class.getName();
+    private final KinesisClientLibConfigurator configurator = new KinesisClientLibConfigurator();
+
+    @Test
+    public void testWithBasicSetup() {
+        MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
+                new String[] {
+                    "streamName = a",
+                    "applicationName = b",
+                    "AwsCredentialsProvider = " + credentialName1,
+                    "workerId = 123"
+                },
+                '\n'));
+        assertEquals(config.getApplicationName(), "b");
+        assertEquals(config.getStreamName(), "a");
+        assertEquals(config.getWorkerIdentifier(), "123");
+        assertThat(config.getMaxGetRecordsThreadPool(), nullValue());
+        assertThat(config.getRetryGetRecordsInSeconds(), nullValue());
+        assertNull(config.getGracefulLeaseHandoffTimeoutMillis());
+        assertNull(config.getIsGracefulLeaseHandoffEnabled());
+    }
+
+    @Test
+    public void testWithLongVariables() {
+        MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
+                new String[] {
+                    "applicationName = app",
+                    "streamName = 123",
+                    "AwsCredentialsProvider = " + credentialName1 + ", " + credentialName2,
+                    "workerId = 123",
+                    "failoverTimeMillis = 100",
+                    "shardSyncIntervalMillis = 500"
+                },
+                '\n'));
+
+        assertEquals(config.getApplicationName(), "app");
+        assertEquals(config.getStreamName(), "123");
+        assertEquals(config.getWorkerIdentifier(), "123");
+        assertEquals(config.getFailoverTimeMillis(), 100);
+        assertEquals(config.getShardSyncIntervalMillis(), 500);
+    }
+
+    @Test
+    public void testWithInitialPositionInStreamExtended() {
+        long epochTimeInSeconds = 1617406032;
+        MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
+                new String[] {
+                    "applicationName = app",
+                    "streamName = 123",
+                    "AwsCredentialsProvider = " + credentialName1 + ", " + credentialName2,
+                    "initialPositionInStreamExtended = " + epochTimeInSeconds
+                },
+                '\n'));
+
+        assertEquals(config.getInitialPositionInStreamExtended().getTimestamp(), new Date(epochTimeInSeconds * 1000L));
+        assertEquals(config.getInitialPositionInStream(), InitialPositionInStream.AT_TIMESTAMP);
+    }
+
+    @Test
+    public void testInvalidInitialPositionInStream() {
+        // AT_TIMESTAMP cannot be used as initialPositionInStream. If a user wants to specify AT_TIMESTAMP,
+        // they must specify the time with initialPositionInStreamExtended.
+        try {
+            getConfiguration(StringUtils.join(
+                    new String[] {
+                        "applicationName = app",
+                        "streamName = 123",
+                        "AwsCredentialsProvider = " + credentialName1 + ", " + credentialName2,
+                        "initialPositionInStream = AT_TIMESTAMP"
+                    },
+                    '\n'));
+            fail("Should have thrown when initialPositionInStream is set to AT_TIMESTAMP");
+        } catch (Exception e) {
+            Throwable rootCause = ExceptionUtils.getRootCause(e);
+            assertTrue(rootCause instanceof IllegalArgumentException);
+        }
+    }
+
+    @Test
+    public void testInvalidInitialPositionInStreamExtended() {
+        // initialPositionInStreamExtended takes a long value indicating seconds since epoch. If a non-long
+        // value is provided, the constructor should throw an IllegalArgumentException exception.
+        try {
+            getConfiguration(StringUtils.join(
+                    new String[] {
+                        "applicationName = app",
+                        "streamName = 123",
+                        "AwsCredentialsProvider = " + credentialName1 + ", " + credentialName2,
+                        "initialPositionInStreamExtended = null"
+                    },
+                    '\n'));
+            fail("Should have thrown when initialPositionInStreamExtended is set to null");
+        } catch (Exception e) {
+            Throwable rootCause = ExceptionUtils.getRootCause(e);
+            assertTrue(rootCause instanceof IllegalArgumentException);
+        }
+    }
+
+    @Test
+    public void testGracefulLeaseHandoffConfig() {
+        final Long testGracefulLeaseHandoffTimeoutMillis = 12345L;
+        final boolean testGracefulLeaseHandoffEnabled = true;
+
+        final MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
+                new String[] {
+                    "applicationName = dummyApplicationName",
+                    "streamName = dummyStreamName",
+                    "AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
+                    "gracefulLeaseHandoffTimeoutMillis = " + testGracefulLeaseHandoffTimeoutMillis,
+                    "isGracefulLeaseHandoffEnabled = " + testGracefulLeaseHandoffEnabled
+                },
+                '\n'));
+
+        assertEquals(testGracefulLeaseHandoffTimeoutMillis, config.getGracefulLeaseHandoffTimeoutMillis());
+        assertEquals(testGracefulLeaseHandoffEnabled, config.getIsGracefulLeaseHandoffEnabled());
+    }
+
+    @Test
+    public void testClientVersionConfig() {
+        final CoordinatorConfig.ClientVersionConfig testClientVersionConfig = Arrays.stream(
+                        CoordinatorConfig.ClientVersionConfig.values())
+                .findAny()
+                .orElseThrow(NoSuchElementException::new);
+
+        final MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
+                new String[] {
+                    "applicationName = dummyApplicationName",
+                    "streamName = dummyStreamName",
+                    "AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
+                    "clientVersionConfig = " + testClientVersionConfig.name()
+                },
+                '\n'));
+
+        assertEquals(testClientVersionConfig, config.getClientVersionConfig());
+    }
+
+    @Test
+    public void testCoordinatorStateConfig() {
+        final String testCoordinatorStateTableName = "CoordState";
+        final BillingMode testCoordinatorStateBillingMode = BillingMode.PAY_PER_REQUEST;
+        final long testCoordinatorStateReadCapacity = 123;
+        final long testCoordinatorStateWriteCapacity = 123;
+
+        final MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
+                new String[] {
+                    "applicationName = dummyApplicationName",
+                    "streamName = dummyStreamName",
+                    "AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
+                    "coordinatorStateTableName = " + testCoordinatorStateTableName,
+                    "coordinatorStateBillingMode = " + testCoordinatorStateBillingMode.name(),
+                    "coordinatorStateReadCapacity = " + testCoordinatorStateReadCapacity,
+                    "coordinatorStateWriteCapacity = " + testCoordinatorStateWriteCapacity
+                },
+                '\n'));
+
+        assertEquals(testCoordinatorStateTableName, config.getCoordinatorStateTableName());
+        assertEquals(testCoordinatorStateBillingMode, config.getCoordinatorStateBillingMode());
+        assertEquals(testCoordinatorStateReadCapacity, config.getCoordinatorStateReadCapacity());
+        assertEquals(testCoordinatorStateWriteCapacity, config.getCoordinatorStateWriteCapacity());
+    }
+
+    @Test
+    public void testWorkerUtilizationAwareAssignmentConfig() {
+        final long testInMemoryWorkerMetricsCaptureFrequencyMillis = 123;
+        final long testWorkerMetricsReporterFreqInMillis = 123;
+        final long testNoOfPersistedMetricsPerWorkerMetrics = 123;
+        final Boolean testDisableWorkerMetrics = true;
+        final double testMaxThroughputPerHostKBps = 123;
+        final long testDampeningPercentage = 12;
+        final long testReBalanceThresholdPercentage = 12;
+        final Boolean testAllowThroughputOvershoot = false;
+        final long testVarianceBalancingFrequency = 12;
+        final double testWorkerMetricsEMAAlpha = .123;
+
+        final MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
+                new String[] {
+                    "applicationName = dummyApplicationName",
+                    "streamName = dummyStreamName",
+                    "AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
+                    "inMemoryWorkerMetricsCaptureFrequencyMillis = " + testInMemoryWorkerMetricsCaptureFrequencyMillis,
+                    "workerMetricsReporterFreqInMillis = " + testWorkerMetricsReporterFreqInMillis,
+                    "noOfPersistedMetricsPerWorkerMetrics = " + testNoOfPersistedMetricsPerWorkerMetrics,
+                    "disableWorkerMetrics = " + testDisableWorkerMetrics,
+                    "maxThroughputPerHostKBps = " + testMaxThroughputPerHostKBps,
+                    "dampeningPercentage = " + testDampeningPercentage,
+                    "reBalanceThresholdPercentage = " + testReBalanceThresholdPercentage,
+                    "allowThroughputOvershoot = " + testAllowThroughputOvershoot,
+                    "varianceBalancingFrequency = " + testVarianceBalancingFrequency,
+                    "workerMetricsEMAAlpha = " + testWorkerMetricsEMAAlpha
+                },
+                '\n'));
+
+        assertEquals(
+                testInMemoryWorkerMetricsCaptureFrequencyMillis,
+                config.getInMemoryWorkerMetricsCaptureFrequencyMillis());
+        assertEquals(testWorkerMetricsReporterFreqInMillis, config.getWorkerMetricsReporterFreqInMillis());
+        assertEquals(testNoOfPersistedMetricsPerWorkerMetrics, config.getNoOfPersistedMetricsPerWorkerMetrics());
+        assertEquals(testDisableWorkerMetrics, config.getDisableWorkerMetrics());
+        assertEquals(testMaxThroughputPerHostKBps, config.getMaxThroughputPerHostKBps(), 0.0001);
+        assertEquals(testDampeningPercentage, config.getDampeningPercentage());
+        assertEquals(testReBalanceThresholdPercentage, config.getReBalanceThresholdPercentage());
+        assertEquals(testAllowThroughputOvershoot, config.getAllowThroughputOvershoot());
+        assertEquals(testVarianceBalancingFrequency, config.getVarianceBalancingFrequency());
+        assertEquals(testWorkerMetricsEMAAlpha, config.getWorkerMetricsEMAAlpha(), 0.0001);
+    }
+
+    @Test
+    public void testWorkerMetricsConfig() {
+        final String testWorkerMetricsTableName = "CoordState";
+        final BillingMode testWorkerMetricsBillingMode = BillingMode.PROVISIONED;
+        final long testWorkerMetricsReadCapacity = 123;
+        final long testWorkerMetricsWriteCapacity = 123;
+
+        final MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
+                new String[] {
+                    "applicationName = dummyApplicationName",
+                    "streamName = dummyStreamName",
+                    "AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
+                    "workerMetricsTableName = " + testWorkerMetricsTableName,
+                    "workerMetricsBillingMode = " + testWorkerMetricsBillingMode.name(),
+                    "workerMetricsReadCapacity = " + testWorkerMetricsReadCapacity,
+                    "workerMetricsWriteCapacity = " + testWorkerMetricsWriteCapacity
+                },
+                '\n'));
+
+        assertEquals(testWorkerMetricsTableName, config.getWorkerMetricsTableName());
+        assertEquals(testWorkerMetricsBillingMode, config.getWorkerMetricsBillingMode());
+        assertEquals(testWorkerMetricsReadCapacity, config.getWorkerMetricsReadCapacity());
+        assertEquals(testWorkerMetricsWriteCapacity, config.getWorkerMetricsWriteCapacity());
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testInvalidClientVersionConfig() {
+        getConfiguration(StringUtils.join(
+                new String[] {
+                    "applicationName = dummyApplicationName",
+                    "streamName = dummyStreamName",
+                    "AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
+                    "clientVersionConfig = " + "invalid_client_version_config"
+                },
+                '\n'));
+    }
+
+    @Test
+    public void testWithUnsupportedClientConfigurationVariables() {
+        MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
+                new String[] {
+                    "AwsCredentialsProvider = " + credentialName1 + ", " + credentialName2,
+                    "workerId = id",
+                    "kinesisClientConfig = {}",
+                    "streamName = stream",
+                    "applicationName = b"
+                },
+                '\n'));
+
+        assertEquals(config.getApplicationName(), "b");
+        assertEquals(config.getStreamName(), "stream");
+        assertEquals(config.getWorkerIdentifier(), "id");
+        // by setting the configuration there is no effect on kinesisClientConfiguration variable.
+    }
+
+    @Test
+    public void testWithIntVariables() {
+        MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
+                new String[] {
+                    "streamName = kinesis",
+                    "AwsCredentialsProvider = " + credentialName2 + ", " + credentialName1,
+                    "workerId = w123",
+                    "maxRecords = 10",
+                    "metricsMaxQueueSize = 20",
+                    "applicationName = kinesis",
+                    "retryGetRecordsInSeconds = 2",
+                    "maxGetRecordsThreadPool = 1"
+                },
+                '\n'));
+
+        assertEquals(config.getApplicationName(), "kinesis");
+        assertEquals(config.getStreamName(), "kinesis");
+        assertEquals(config.getWorkerIdentifier(), "w123");
+        assertEquals(config.getMaxRecords(), 10);
+        assertEquals(config.getMetricsMaxQueueSize(), 20);
+        assertThat(config.getRetryGetRecordsInSeconds(), equalTo(2));
+        assertThat(config.getMaxGetRecordsThreadPool(), equalTo(1));
+    }
+
+    @Test
+    public void testWithBooleanVariables() {
+        MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
+                new String[] {
+                    "streamName = a",
+                    "applicationName = b",
+                    "AwsCredentialsProvider = ABCD, " + credentialName1,
+                    "workerId = 0",
+                    "cleanupLeasesUponShardCompletion = false",
+                    "validateSequenceNumberBeforeCheckpointing = true"
+                },
+                '\n'));
+
+        assertEquals(config.getApplicationName(), "b");
+        assertEquals(config.getStreamName(), "a");
+        assertEquals(config.getWorkerIdentifier(), "0");
+        assertFalse(config.isCleanupLeasesUponShardCompletion());
+        assertTrue(config.isValidateSequenceNumberBeforeCheckpointing());
+    }
+
+    @Test
+    public void testWithStringVariables() {
+        MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
+                new String[] {
+                    "streamName = a",
+                    "applicationName = b",
+                    "AwsCredentialsProvider = ABCD," + credentialName1,
+                    "workerId = 1",
+                    "kinesisEndpoint = https://kinesis",
+                    "metricsLevel = SUMMARY"
+                },
+                '\n'));
+
+        assertEquals(config.getWorkerIdentifier(), "1");
+        assertEquals(config.getKinesisClient().get("endpointOverride"), URI.create("https://kinesis"));
+        assertEquals(config.getMetricsLevel(), MetricsLevel.SUMMARY);
+    }
+
+    @Test
+    public void testWithSetVariables() {
+        MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
+                new String[] {
+                    "streamName = a",
+                    "applicationName = b",
+                    "AwsCredentialsProvider = ABCD," + credentialName1,
+                    "workerId = 1",
+                    "metricsEnabledDimensions = ShardId, WorkerIdentifier"
+                },
+                '\n'));
+
+        Set<String> expectedMetricsEnabledDimensions = ImmutableSet.<String>builder()
+                .add("ShardId", "WorkerIdentifier")
+                .build();
+        assertThat(
+                new HashSet<>(Arrays.asList(config.getMetricsEnabledDimensions())),
+                equalTo(expectedMetricsEnabledDimensions));
+    }
+
+    @Test
+    public void testWithInitialPositionInStreamTrimHorizon() {
+        MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
+                new String[] {
+                    "streamName = a",
+                    "applicationName = b",
+                    "AwsCredentialsProvider = ABCD," + credentialName1,
+                    "workerId = 123",
+                    "initialPositionInStream = TriM_Horizon"
+                },
+                '\n'));
+
+        assertEquals(config.getInitialPositionInStream(), InitialPositionInStream.TRIM_HORIZON);
+    }
+
+    @Test
+    public void testWithInitialPositionInStreamLatest() {
+        MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
+                new String[] {
+                    "streamName = a",
+                    "applicationName = b",
+                    "AwsCredentialsProvider = ABCD," + credentialName1,
+                    "workerId = 123",
+                    "initialPositionInStream = LateSt"
+                },
+                '\n'));
+
+        assertEquals(config.getInitialPositionInStream(), InitialPositionInStream.LATEST);
+    }
+
+    @Test
+    public void testSkippingNonKCLVariables() {
+        MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
+                new String[] {
+                    "streamName = a",
+                    "applicationName = b",
+                    "AwsCredentialsProvider = ABCD," + credentialName1,
+                    "workerId = 123",
+                    "initialPositionInStream = TriM_Horizon",
+                    "abc = 1"
+                },
+                '\n'));
+
+        assertEquals(config.getApplicationName(), "b");
+        assertEquals(config.getStreamName(), "a");
+        assertEquals(config.getWorkerIdentifier(), "123");
+        assertEquals(config.getInitialPositionInStream(), InitialPositionInStream.TRIM_HORIZON);
+    }
+
+    @Test
+    public void testEmptyOptionalVariables() {
+        MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
+                new String[] {
+                    "streamName = a",
+                    "applicationName = b",
+                    "AwsCredentialsProvider = ABCD," + credentialName1,
+                    "workerId = 123",
+                    "initialPositionInStream = TriM_Horizon",
+                    "maxGetRecordsThreadPool = 1"
+                },
+                '\n'));
+        assertThat(config.getMaxGetRecordsThreadPool(), equalTo(1));
+        assertThat(config.getRetryGetRecordsInSeconds(), nullValue());
+    }
+
+    @Test
+    public void testWithZeroValue() {
+        String test = StringUtils.join(
+                new String[] {
+                    "streamName = a",
+                    "applicationName = b",
+                    "AwsCredentialsProvider = ABCD," + credentialName1,
+                    "workerId = 123",
+                    "initialPositionInStream = TriM_Horizon",
+                    "maxGetRecordsThreadPool = 0",
+                    "retryGetRecordsInSeconds = 0"
+                },
+                '\n');
+        getConfiguration(test);
+    }
+
+    @Test
+    public void testWithInvalidIntValue() {
+        String test = StringUtils.join(
+                new String[] {
+                    "streamName = a",
+                    "applicationName = b",
+                    "AwsCredentialsProvider = " + credentialName1,
+                    "workerId = 123",
+                    "failoverTimeMillis = 100nf"
+                },
+                '\n');
+        getConfiguration(test);
+    }
+
+    @Test
+    public void testWithNegativeIntValue() {
+        String test = StringUtils.join(
+                new String[] {
+                    "streamName = a",
+                    "applicationName = b",
+                    "AwsCredentialsProvider = " + credentialName1,
+                    "workerId = 123",
+                    "failoverTimeMillis = -12"
+                },
+                '\n');
+
+        // separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement
+        getConfiguration(test);
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testWithMissingCredentialsProvider() {
+        String test = StringUtils.join(
+                new String[] {
+                    "streamName = a",
+                    "applicationName = b",
+                    "workerId = 123",
+                    "failoverTimeMillis = 100",
+                    "shardSyncIntervalMillis = 500"
+                },
+                '\n');
+
+        // separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement
+        getConfiguration(test);
+    }
+
+    @Test
+    public void testWithMissingWorkerId() {
+        String test = StringUtils.join(
+                new String[] {
+                    "streamName = a",
+                    "applicationName = b",
+                    "AwsCredentialsProvider = " + credentialName1,
+                    "failoverTimeMillis = 100",
+                    "shardSyncIntervalMillis = 500"
+                },
+                '\n');
+        MultiLangDaemonConfiguration config = getConfiguration(test);
+
+        // if workerId is not provided, configurator should assign one for it automatically
+        assertNotNull(config.getWorkerIdentifier());
+        assertFalse(config.getWorkerIdentifier().isEmpty());
+    }
+
+    @Test(expected = NullPointerException.class)
+    public void testWithMissingStreamNameAndMissingStreamArn() {
+        String test = StringUtils.join(
+                new String[] {
+                    "applicationName = b",
+                    "AwsCredentialsProvider = " + credentialName1,
+                    "workerId = 123",
+                    "failoverTimeMillis = 100"
+                },
+                '\n');
+        getConfiguration(test);
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testWithEmptyStreamNameAndMissingStreamArn() {
+        String test = StringUtils.join(
+                new String[] {
+                    "applicationName = b",
+                    "AwsCredentialsProvider = " + credentialName1,
+                    "workerId = 123",
+                    "failoverTimeMillis = 100",
+                    "streamName = ",
+                    "streamArn = "
+                },
+                '\n');
+        getConfiguration(test);
+    }
+
+    @Test(expected = NullPointerException.class)
+    public void testWithMissingApplicationName() {
+        String test = StringUtils.join(
+                new String[] {
+                    "streamName = a",
+                    "AwsCredentialsProvider = " + credentialName1,
+                    "workerId = 123",
+                    "failoverTimeMillis = 100"
+                },
+                '\n');
+        getConfiguration(test);
+    }
+
+    @Test
+    public void testWithAwsCredentialsFailed() {
+        String test = StringUtils.join(
+                new String[] {
+                    "streamName = a",
+                    "applicationName = b",
+                    "AwsCredentialsProvider = " + credentialName2,
+                    "failoverTimeMillis = 100",
+                    "shardSyncIntervalMillis = 500"
+                },
+                '\n');
+        MultiLangDaemonConfiguration config = getConfiguration(test);
+
+        // separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement
+        try {
+            config.getKinesisCredentialsProvider()
+                    .build(AwsCredentialsProvider.class)
+                    .resolveCredentials();
+            fail("expect failure with wrong credentials provider");
+        } catch (Exception e) {
+            // succeed
+        }
+    }
+
+    @Test
+    public void testProcessKeyWithExpectedCasing() {
+        String key = "AwsCredentialsProvider";
+        String result = configurator.processKey(key);
+        assertEquals("awsCredentialsProvider", result);
+    }
+
+    @Test
+    public void testProcessKeyWithOldCasing() {
+        String key = "AWSCredentialsProvider";
+        String result = configurator.processKey(key);
+        assertEquals("awsCredentialsProvider", result);
+    }
+
+    @Test
+    public void testProcessKeyWithMixedCasing() {
+        String key = "AwScReDeNtIaLsPrOvIdEr";
+        String result = configurator.processKey(key);
+        assertEquals("awsCredentialsProvider", result);
+    }
+
+    @Test
+    public void testProcessKeyWithSuffix() {
+        String key = "awscredentialsproviderDynamoDB";
+        String result = configurator.processKey(key);
+        assertEquals("awsCredentialsProviderDynamoDB", result);
+    }
+
+    // TODO: fix this test
+    @Test
+    public void testWithDifferentAwsCredentialsForDynamoDBAndCloudWatch() {
+        String test = StringUtils.join(
+                new String[] {
+                    "streamName = a",
+                    "applicationName = b",
+                    "AwsCredentialsProvider = " + credentialNameKinesis,
+                    "AwsCredentialsProviderDynamoDB = " + credentialNameDynamoDB,
+                    "AwsCredentialsProviderCloudWatch = " + credentialNameCloudWatch,
+                    "failoverTimeMillis = 100",
+                    "shardSyncIntervalMillis = 500"
+                },
+                '\n');
+
+        // separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement
+        final MultiLangDaemonConfiguration config = getConfiguration(test);
+        config.getKinesisCredentialsProvider()
+                .build(AwsCredentialsProvider.class)
+                .resolveCredentials();
+        config.getDynamoDBCredentialsProvider()
+                .build(AwsCredentialsProvider.class)
+                .resolveCredentials();
+        config.getCloudWatchCredentialsProvider()
+                .build(AwsCredentialsProvider.class)
+                .resolveCredentials();
+    }
+
+    // TODO: fix this test
+    @Test
+    public void testWithDifferentAwsCredentialsForDynamoDBAndCloudWatchFailed() {
+        String test = StringUtils.join(
+                new String[] {
+                    "streamName = a",
+                    "applicationName = b",
+                    "AwsCredentialsProvider = " + credentialNameKinesis,
+                    "AwsCredentialsProviderDynamoDB = " + credentialName2,
+                    "AwsCredentialsProviderCloudWatch = " + credentialName2,
+                    "failoverTimeMillis = 100",
+                    "shardSyncIntervalMillis = 500"
+                },
+                '\n');
+
+        // separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement
+        final MultiLangDaemonConfiguration config = getConfiguration(test);
+        config.getKinesisCredentialsProvider()
+                .build(AwsCredentialsProvider.class)
+                .resolveCredentials();
+        try {
+            config.getDynamoDBCredentialsProvider()
+                    .build(AwsCredentialsProvider.class)
+                    .resolveCredentials();
+            fail("DynamoDB credential providers should fail.");
+        } catch (Exception e) {
+            // succeed
+        }
+        try {
+            config.getCloudWatchCredentialsProvider()
+                    .build(AwsCredentialsProvider.class)
+                    .resolveCredentials();
+            fail("CloudWatch credential providers should fail.");
+        } catch (Exception e) {
+            // succeed
+        }
+    }
+
+    /**
+     * This credentials provider will always succeed
+     */
+    public static class AlwaysSucceedCredentialsProvider implements AwsCredentialsProvider {
+        @Override
+        public AwsCredentials resolveCredentials() {
+            return AwsBasicCredentials.create("a", "b");
+        }
+    }
+
+    /**
+     * This credentials provider will always succeed
+     */
+    public static class AlwaysSucceedCredentialsProviderKinesis implements AwsCredentialsProvider {
+        @Override
+        public AwsCredentials resolveCredentials() {
+            return AwsBasicCredentials.create("DUMMY_ACCESS_KEY_ID", "DUMMY_SECRET_ACCESS_KEY");
+        }
+    }
+
+    /**
+     * This credentials provider will always succeed
+     */
+    public static class AlwaysSucceedCredentialsProviderDynamoDB implements AwsCredentialsProvider {
+        @Override
+        public AwsCredentials resolveCredentials() {
+            return AwsBasicCredentials.create("DUMMY_ACCESS_KEY_ID", "DUMMY_SECRET_ACCESS_KEY");
+        }
+    }
+
+    /**
+     * This credentials provider will always succeed
+     */
+    public static class AlwaysSucceedCredentialsProviderCloudWatch implements AwsCredentialsProvider {
+        @Override
+        public AwsCredentials resolveCredentials() {
+            return AwsBasicCredentials.create("DUMMY_ACCESS_KEY_ID", "DUMMY_SECRET_ACCESS_KEY");
+        }
+    }
+
+    /**
+     * This credentials provider will always fail
+     */
+    public static class AlwaysFailCredentialsProvider implements AwsCredentialsProvider {
+
+        @Override
+        public AwsCredentials resolveCredentials() {
+            throw new IllegalArgumentException();
+        }
+    }
+
+    private MultiLangDaemonConfiguration getConfiguration(String configString) {
+        InputStream input = new ByteArrayInputStream(configString.getBytes());
+        return configurator.getConfiguration(input);
+    }
+}

amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/MultiLangDaemonConfigurationTest.java

@@ -0,0 +1,513 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.multilang.config;
+
+import java.util.Arrays;
+import java.util.NoSuchElementException;
+
+import org.apache.commons.beanutils.BeanUtilsBean;
+import org.apache.commons.beanutils.ConvertUtilsBean;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.runners.MockitoJUnitRunner;
+import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
+import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient;
+import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
+import software.amazon.awssdk.services.dynamodb.model.BillingMode;
+import software.amazon.awssdk.services.kinesis.KinesisAsyncClient;
+import software.amazon.kinesis.common.ConfigsBuilder;
+import software.amazon.kinesis.coordinator.CoordinatorConfig;
+import software.amazon.kinesis.leases.LeaseManagementConfig;
+import software.amazon.kinesis.processor.ShardRecordProcessorFactory;
+import software.amazon.kinesis.retrieval.fanout.FanOutConfig;
+import software.amazon.kinesis.retrieval.polling.PollingConfig;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(MockitoJUnitRunner.class)
+public class MultiLangDaemonConfigurationTest {
+
+    private static final String AWS_REGION_PROPERTY_NAME = "aws.region";
+    private static final String DUMMY_APPLICATION_NAME = "dummyApplicationName";
+    private static final String DUMMY_STREAM_NAME = "dummyStreamName";
+
+    private BeanUtilsBean utilsBean;
+    private ConvertUtilsBean convertUtilsBean;
+    private String originalRegionValue;
+
+    @Mock
+    private ShardRecordProcessorFactory shardRecordProcessorFactory;
+
+    @Rule
+    public final ExpectedException thrown = ExpectedException.none();
+
+    @Before
+    public void setup() {
+        originalRegionValue = System.getProperty(AWS_REGION_PROPERTY_NAME);
+        System.setProperty(AWS_REGION_PROPERTY_NAME, "us-east-1");
+        convertUtilsBean = new ConvertUtilsBean();
+        utilsBean = new BeanUtilsBean(convertUtilsBean);
+    }
+
+    @After
+    public void after() {
+        if (originalRegionValue != null) {
+            System.setProperty(AWS_REGION_PROPERTY_NAME, originalRegionValue);
+        } else {
+            System.clearProperty(AWS_REGION_PROPERTY_NAME);
+        }
+    }
+
+    public MultiLangDaemonConfiguration baseConfiguration() {
+        MultiLangDaemonConfiguration configuration = new MultiLangDaemonConfiguration(utilsBean, convertUtilsBean);
+        configuration.setApplicationName(DUMMY_APPLICATION_NAME);
+        configuration.setStreamName(DUMMY_STREAM_NAME);
+        configuration.getKinesisCredentialsProvider().set("class", DefaultCredentialsProvider.class.getName());
+
+        return configuration;
+    }
+
+    @Test
+    public void testSetPrimitiveValue() {
+        MultiLangDaemonConfiguration configuration = baseConfiguration();
+        configuration.setMaxLeasesForWorker(10);
+
+        MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
+                configuration.resolvedConfiguration(shardRecordProcessorFactory);
+
+        assertThat(resolvedConfiguration.leaseManagementConfig.maxLeasesForWorker(), equalTo(10));
+    }
+
+    @Test
+    public void testSetEnablePriorityLeaseAssignment() {
+        MultiLangDaemonConfiguration configuration = baseConfiguration();
+        configuration.setEnablePriorityLeaseAssignment(false);
+
+        MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
+                configuration.resolvedConfiguration(shardRecordProcessorFactory);
+
+        assertThat(resolvedConfiguration.leaseManagementConfig.enablePriorityLeaseAssignment(), equalTo(false));
+    }
+
+    @Test
+    public void testSetLeaseTableDeletionProtectionEnabledToTrue() {
+        MultiLangDaemonConfiguration configuration = baseConfiguration();
+        configuration.setLeaseTableDeletionProtectionEnabled(true);
+
+        MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
+                configuration.resolvedConfiguration(shardRecordProcessorFactory);
+
+        assertTrue(resolvedConfiguration.leaseManagementConfig.leaseTableDeletionProtectionEnabled());
+    }
+
+    @Test
+    public void testGracefulLeaseHandoffConfig() {
+        final LeaseManagementConfig.GracefulLeaseHandoffConfig defaultGracefulLeaseHandoffConfig =
+                getTestConfigsBuilder().leaseManagementConfig().gracefulLeaseHandoffConfig();
+
+        final long testGracefulLeaseHandoffTimeoutMillis =
+                defaultGracefulLeaseHandoffConfig.gracefulLeaseHandoffTimeoutMillis() + 12345;
+        final boolean testGracefulLeaseHandoffEnabled =
+                !defaultGracefulLeaseHandoffConfig.isGracefulLeaseHandoffEnabled();
+
+        final MultiLangDaemonConfiguration configuration = baseConfiguration();
+        configuration.setGracefulLeaseHandoffTimeoutMillis(testGracefulLeaseHandoffTimeoutMillis);
+        configuration.setIsGracefulLeaseHandoffEnabled(testGracefulLeaseHandoffEnabled);
+
+        final MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
+                configuration.resolvedConfiguration(shardRecordProcessorFactory);
+
+        final LeaseManagementConfig.GracefulLeaseHandoffConfig gracefulLeaseHandoffConfig =
+                resolvedConfiguration.leaseManagementConfig.gracefulLeaseHandoffConfig();
+
+        assertEquals(
+                testGracefulLeaseHandoffTimeoutMillis, gracefulLeaseHandoffConfig.gracefulLeaseHandoffTimeoutMillis());
+        assertEquals(testGracefulLeaseHandoffEnabled, gracefulLeaseHandoffConfig.isGracefulLeaseHandoffEnabled());
+    }
+
+    @Test
+    public void testGracefulLeaseHandoffUsesDefaults() {
+        final MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
+                baseConfiguration().resolvedConfiguration(shardRecordProcessorFactory);
+
+        final LeaseManagementConfig.GracefulLeaseHandoffConfig gracefulLeaseHandoffConfig =
+                resolvedConfiguration.leaseManagementConfig.gracefulLeaseHandoffConfig();
+
+        final LeaseManagementConfig.GracefulLeaseHandoffConfig defaultGracefulLeaseHandoffConfig =
+                getTestConfigsBuilder().leaseManagementConfig().gracefulLeaseHandoffConfig();
+
+        assertEquals(defaultGracefulLeaseHandoffConfig, gracefulLeaseHandoffConfig);
+    }
+
+    @Test
+    public void testWorkerUtilizationAwareAssignmentConfig() {
+        MultiLangDaemonConfiguration configuration = baseConfiguration();
+
+        configuration.setInMemoryWorkerMetricsCaptureFrequencyMillis(123);
+        configuration.setWorkerMetricsReporterFreqInMillis(123);
+        configuration.setNoOfPersistedMetricsPerWorkerMetrics(123);
+        configuration.setDisableWorkerMetrics(true);
+        configuration.setMaxThroughputPerHostKBps(.123);
+        configuration.setDampeningPercentage(12);
+        configuration.setReBalanceThresholdPercentage(12);
+        configuration.setAllowThroughputOvershoot(false);
+        configuration.setVarianceBalancingFrequency(12);
+        configuration.setWorkerMetricsEMAAlpha(.123);
+
+        MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
+                configuration.resolvedConfiguration(shardRecordProcessorFactory);
+        LeaseManagementConfig leaseManagementConfig = resolvedConfiguration.leaseManagementConfig;
+        LeaseManagementConfig.WorkerUtilizationAwareAssignmentConfig config =
+                leaseManagementConfig.workerUtilizationAwareAssignmentConfig();
+
+        assertEquals(config.inMemoryWorkerMetricsCaptureFrequencyMillis(), 123);
+        assertEquals(config.workerMetricsReporterFreqInMillis(), 123);
+        assertEquals(config.noOfPersistedMetricsPerWorkerMetrics(), 123);
+        assertTrue(config.disableWorkerMetrics());
+        assertEquals(config.maxThroughputPerHostKBps(), .123, .25);
+        assertEquals(config.dampeningPercentage(), 12);
+        assertEquals(config.reBalanceThresholdPercentage(), 12);
+        assertFalse(config.allowThroughputOvershoot());
+        assertEquals(config.varianceBalancingFrequency(), 12);
+        assertEquals(config.workerMetricsEMAAlpha(), .123, .25);
+    }
+
+    @Test
+    public void testWorkerUtilizationAwareAssignmentConfigUsesDefaults() {
+        final LeaseManagementConfig.WorkerUtilizationAwareAssignmentConfig defaultWorkerUtilAwareAssignmentConfig =
+                getTestConfigsBuilder().leaseManagementConfig().workerUtilizationAwareAssignmentConfig();
+
+        final MultiLangDaemonConfiguration configuration = baseConfiguration();
+        configuration.setVarianceBalancingFrequency(
+                defaultWorkerUtilAwareAssignmentConfig.varianceBalancingFrequency() + 12345);
+
+        final MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
+                configuration.resolvedConfiguration(shardRecordProcessorFactory);
+
+        final LeaseManagementConfig.WorkerUtilizationAwareAssignmentConfig resolvedWorkerUtilAwareAssignmentConfig =
+                resolvedConfiguration.leaseManagementConfig.workerUtilizationAwareAssignmentConfig();
+
+        assertNotEquals(defaultWorkerUtilAwareAssignmentConfig, resolvedWorkerUtilAwareAssignmentConfig);
+
+        // apart from the single updated configuration, all other config values should be equal to the default
+        resolvedWorkerUtilAwareAssignmentConfig.varianceBalancingFrequency(
+                defaultWorkerUtilAwareAssignmentConfig.varianceBalancingFrequency());
+        assertEquals(defaultWorkerUtilAwareAssignmentConfig, resolvedWorkerUtilAwareAssignmentConfig);
+    }
+
+    @Test
+    public void testWorkerMetricsTableConfigBean() {
+        final BillingMode testWorkerMetricsTableBillingMode = BillingMode.PROVISIONED;
+
+        MultiLangDaemonConfiguration configuration = baseConfiguration();
+
+        configuration.setWorkerMetricsTableName("testTable");
+        configuration.setWorkerMetricsBillingMode(testWorkerMetricsTableBillingMode);
+        configuration.setWorkerMetricsReadCapacity(123);
+        configuration.setWorkerMetricsWriteCapacity(123);
+
+        MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
+                configuration.resolvedConfiguration(shardRecordProcessorFactory);
+        LeaseManagementConfig leaseManagementConfig = resolvedConfiguration.leaseManagementConfig;
+        LeaseManagementConfig.WorkerUtilizationAwareAssignmentConfig workerUtilizationConfig =
+                leaseManagementConfig.workerUtilizationAwareAssignmentConfig();
+        LeaseManagementConfig.WorkerMetricsTableConfig workerMetricsConfig =
+                workerUtilizationConfig.workerMetricsTableConfig();
+
+        assertEquals(workerMetricsConfig.tableName(), "testTable");
+        assertEquals(workerMetricsConfig.billingMode(), testWorkerMetricsTableBillingMode);
+        assertEquals(workerMetricsConfig.readCapacity(), 123);
+        assertEquals(workerMetricsConfig.writeCapacity(), 123);
+    }
+
+    @Test
+    public void testWorkerMetricsTableConfigUsesDefaults() {
+        final LeaseManagementConfig.WorkerMetricsTableConfig defaultWorkerMetricsTableConfig = getTestConfigsBuilder()
+                .leaseManagementConfig()
+                .workerUtilizationAwareAssignmentConfig()
+                .workerMetricsTableConfig();
+
+        final MultiLangDaemonConfiguration configuration = baseConfiguration();
+        configuration.setWorkerMetricsBillingMode(Arrays.stream(BillingMode.values())
+                .filter(billingMode -> billingMode != defaultWorkerMetricsTableConfig.billingMode())
+                .findFirst()
+                .orElseThrow(NoSuchElementException::new));
+
+        final MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
+                configuration.resolvedConfiguration(shardRecordProcessorFactory);
+
+        final LeaseManagementConfig.WorkerMetricsTableConfig resolvedWorkerMetricsTableConfig = resolvedConfiguration
+                .leaseManagementConfig
+                .workerUtilizationAwareAssignmentConfig()
+                .workerMetricsTableConfig();
+
+        assertNotEquals(defaultWorkerMetricsTableConfig, resolvedWorkerMetricsTableConfig);
+
+        // apart from the single updated configuration, all other config values should be equal to the default
+        resolvedWorkerMetricsTableConfig.billingMode(defaultWorkerMetricsTableConfig.billingMode());
+        assertEquals(defaultWorkerMetricsTableConfig, resolvedWorkerMetricsTableConfig);
+    }
+
+    @Test
+    public void testCoordinatorStateTableConfigBean() {
+        final BillingMode testWorkerMetricsTableBillingMode = BillingMode.PAY_PER_REQUEST;
+
+        MultiLangDaemonConfiguration configuration = baseConfiguration();
+
+        configuration.setCoordinatorStateTableName("testTable");
+        configuration.setCoordinatorStateBillingMode(testWorkerMetricsTableBillingMode);
+        configuration.setCoordinatorStateReadCapacity(123);
+        configuration.setCoordinatorStateWriteCapacity(123);
+
+        MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
+                configuration.resolvedConfiguration(shardRecordProcessorFactory);
+        CoordinatorConfig coordinatorConfig = resolvedConfiguration.getCoordinatorConfig();
+        CoordinatorConfig.CoordinatorStateTableConfig coordinatorStateConfig =
+                coordinatorConfig.coordinatorStateTableConfig();
+        assertEquals(coordinatorStateConfig.tableName(), "testTable");
+        assertEquals(coordinatorStateConfig.billingMode(), testWorkerMetricsTableBillingMode);
+        assertEquals(coordinatorStateConfig.readCapacity(), 123);
+        assertEquals(coordinatorStateConfig.writeCapacity(), 123);
+    }
+
+    @Test
+    public void testCoordinatorStateTableConfigUsesDefaults() {
+        final CoordinatorConfig.CoordinatorStateTableConfig defaultCoordinatorStateTableConfig =
+                getTestConfigsBuilder().coordinatorConfig().coordinatorStateTableConfig();
+
+        final MultiLangDaemonConfiguration configuration = baseConfiguration();
+        configuration.setCoordinatorStateWriteCapacity(defaultCoordinatorStateTableConfig.writeCapacity() + 12345);
+
+        final MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
+                configuration.resolvedConfiguration(shardRecordProcessorFactory);
+
+        final CoordinatorConfig.CoordinatorStateTableConfig resolvedCoordinatorStateTableConfig =
+                resolvedConfiguration.coordinatorConfig.coordinatorStateTableConfig();
+
+        assertNotEquals(defaultCoordinatorStateTableConfig, resolvedCoordinatorStateTableConfig);
+
+        // apart from the single updated configuration, all other config values should be equal to the default
+        resolvedCoordinatorStateTableConfig.writeCapacity(defaultCoordinatorStateTableConfig.writeCapacity());
+        assertEquals(defaultCoordinatorStateTableConfig, resolvedCoordinatorStateTableConfig);
+    }
+
+    @Test
+    public void testSetLeaseTablePitrEnabledToTrue() {
+        MultiLangDaemonConfiguration configuration = baseConfiguration();
+        configuration.setLeaseTablePitrEnabled(true);
+
+        MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
+                configuration.resolvedConfiguration(shardRecordProcessorFactory);
+
+        assertTrue(resolvedConfiguration.leaseManagementConfig.leaseTablePitrEnabled());
+    }
+
+    @Test
+    public void testSetLeaseTableDeletionProtectionEnabledToFalse() {
+        MultiLangDaemonConfiguration configuration = baseConfiguration();
+        configuration.setLeaseTableDeletionProtectionEnabled(false);
+
+        MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
+                configuration.resolvedConfiguration(shardRecordProcessorFactory);
+
+        assertFalse(resolvedConfiguration.leaseManagementConfig.leaseTableDeletionProtectionEnabled());
+    }
+
+    @Test
+    public void testSetLeaseTablePitrEnabledToFalse() {
+        MultiLangDaemonConfiguration configuration = baseConfiguration();
+        configuration.setLeaseTablePitrEnabled(false);
+
+        MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
+                configuration.resolvedConfiguration(shardRecordProcessorFactory);
+
+        assertFalse(resolvedConfiguration.leaseManagementConfig.leaseTablePitrEnabled());
+    }
+
+    @Test
+    public void testDefaultRetrievalConfig() {
+        MultiLangDaemonConfiguration configuration = baseConfiguration();
+
+        MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
+                configuration.resolvedConfiguration(shardRecordProcessorFactory);
+
+        assertThat(
+                resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), instanceOf(FanOutConfig.class));
+    }
+
+    @Test
+    public void testDefaultRetrievalConfigWithPollingConfigSet() {
+        MultiLangDaemonConfiguration configuration = baseConfiguration();
+        configuration.setMaxRecords(10);
+        configuration.setIdleTimeBetweenReadsInMillis(60000);
+
+        MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
+                configuration.resolvedConfiguration(shardRecordProcessorFactory);
+
+        assertThat(
+                resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), instanceOf(PollingConfig.class));
+        assertEquals(
+                10,
+                ((PollingConfig) resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig()).maxRecords());
+        assertEquals(
+                60000,
+                ((PollingConfig) resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig())
+                        .idleTimeBetweenReadsInMillis());
+        assertTrue(((PollingConfig) resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig())
+                .usePollingConfigIdleTimeValue());
+    }
+
+    @Test
+    public void testFanoutRetrievalMode() {
+        MultiLangDaemonConfiguration configuration = baseConfiguration();
+        configuration.setRetrievalMode(RetrievalMode.FANOUT);
+
+        MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
+                configuration.resolvedConfiguration(shardRecordProcessorFactory);
+
+        assertThat(
+                resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), instanceOf(FanOutConfig.class));
+    }
+
+    @Test
+    public void testPollingRetrievalMode() {
+        MultiLangDaemonConfiguration configuration = baseConfiguration();
+        configuration.setRetrievalMode(RetrievalMode.POLLING);
+
+        MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
+                configuration.resolvedConfiguration(shardRecordProcessorFactory);
+
+        assertThat(
+                resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), instanceOf(PollingConfig.class));
+    }
+
+    @Test
+    public void testRetrievalModeSetForPollingString() throws Exception {
+        MultiLangDaemonConfiguration configuration = baseConfiguration();
+
+        utilsBean.setProperty(
+                configuration, "retrievalMode", RetrievalMode.POLLING.name().toLowerCase());
+
+        MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
+                configuration.resolvedConfiguration(shardRecordProcessorFactory);
+
+        assertThat(
+                resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), instanceOf(PollingConfig.class));
+    }
+
+    @Test
+    public void testRetrievalModeSetForFanoutString() throws Exception {
+        MultiLangDaemonConfiguration configuration = baseConfiguration();
+
+        utilsBean.setProperty(
+                configuration, "retrievalMode", RetrievalMode.FANOUT.name().toLowerCase());
+
+        MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
+                configuration.resolvedConfiguration(shardRecordProcessorFactory);
+
+        assertThat(
+                resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), instanceOf(FanOutConfig.class));
+    }
+
+    @Test
+    public void testInvalidRetrievalMode() throws Exception {
+        thrown.expect(IllegalArgumentException.class);
+        thrown.expectMessage("Unknown retrieval type");
+
+        MultiLangDaemonConfiguration configuration = baseConfiguration();
+
+        utilsBean.setProperty(configuration, "retrievalMode", "invalid");
+    }
+
+    // @Test
+    // TODO : Enable this test once https://github.com/awslabs/amazon-kinesis-client/issues/692 is resolved
+    public void testmetricsEnabledDimensions() {
+        MultiLangDaemonConfiguration configuration = baseConfiguration();
+        configuration.setMetricsEnabledDimensions(new String[] {"Operation"});
+        configuration.resolvedConfiguration(shardRecordProcessorFactory);
+    }
+
+    @Test
+    public void testFanoutConfigSetConsumerName() {
+        String consumerArn = "test-consumer";
+
+        MultiLangDaemonConfiguration configuration = baseConfiguration();
+
+        configuration.setRetrievalMode(RetrievalMode.FANOUT);
+        configuration.getFanoutConfig().setConsumerArn(consumerArn);
+
+        MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
+                configuration.resolvedConfiguration(shardRecordProcessorFactory);
+
+        assertThat(
+                resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), instanceOf(FanOutConfig.class));
+        FanOutConfig fanOutConfig =
+                (FanOutConfig) resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig();
+
+        assertThat(fanOutConfig.consumerArn(), equalTo(consumerArn));
+    }
+
+    @Test
+    public void testClientVersionConfig() {
+        final CoordinatorConfig.ClientVersionConfig testClientVersionConfig =
+                CoordinatorConfig.ClientVersionConfig.CLIENT_VERSION_CONFIG_COMPATIBLE_WITH_2X;
+
+        final MultiLangDaemonConfiguration configuration = baseConfiguration();
+        configuration.setClientVersionConfig(testClientVersionConfig);
+
+        final MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
+                configuration.resolvedConfiguration(shardRecordProcessorFactory);
+
+        final CoordinatorConfig coordinatorConfig = resolvedConfiguration.coordinatorConfig;
+
+        assertEquals(testClientVersionConfig, coordinatorConfig.clientVersionConfig());
+    }
+
+    @Test
+    public void testClientVersionConfigUsesDefault() {
+        final MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
+                baseConfiguration().resolvedConfiguration(shardRecordProcessorFactory);
+
+        final CoordinatorConfig coordinatorConfig = resolvedConfiguration.coordinatorConfig;
+
+        assertEquals(
+                getTestConfigsBuilder().coordinatorConfig().clientVersionConfig(),
+                coordinatorConfig.clientVersionConfig());
+    }
+
+    private ConfigsBuilder getTestConfigsBuilder() {
+        return new ConfigsBuilder(
+                DUMMY_STREAM_NAME,
+                DUMMY_APPLICATION_NAME,
+                Mockito.mock(KinesisAsyncClient.class),
+                Mockito.mock(DynamoDbAsyncClient.class),
+                Mockito.mock(CloudWatchAsyncClient.class),
+                "dummyWorkerIdentifier",
+                shardRecordProcessorFactory);
+    }
+}

amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/PollingConfigBeanTest.java

@@ -0,0 +1,68 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.multilang.config;
+
+import java.util.Optional;
+
+import org.apache.commons.beanutils.BeanUtilsBean;
+import org.apache.commons.beanutils.ConvertUtilsBean;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.runners.MockitoJUnitRunner;
+import software.amazon.awssdk.services.kinesis.KinesisAsyncClient;
+import software.amazon.kinesis.retrieval.polling.PollingConfig;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.junit.Assert.assertThat;
+
+@RunWith(MockitoJUnitRunner.class)
+public class PollingConfigBeanTest {
+
+    @Mock
+    private KinesisAsyncClient kinesisAsyncClient;
+
+    @Test
+    public void testAllPropertiesTransit() {
+        PollingConfigBean pollingConfigBean = new PollingConfigBean();
+        pollingConfigBean.setIdleTimeBetweenReadsInMillis(1000);
+        pollingConfigBean.setMaxGetRecordsThreadPool(20);
+        pollingConfigBean.setMaxRecords(5000);
+        pollingConfigBean.setRetryGetRecordsInSeconds(30);
+
+        ConvertUtilsBean convertUtilsBean = new ConvertUtilsBean();
+        BeanUtilsBean utilsBean = new BeanUtilsBean(convertUtilsBean);
+
+        MultiLangDaemonConfiguration multiLangDaemonConfiguration =
+                new MultiLangDaemonConfiguration(utilsBean, convertUtilsBean);
+        multiLangDaemonConfiguration.setStreamName("test-stream");
+
+        PollingConfig pollingConfig = pollingConfigBean.build(kinesisAsyncClient, multiLangDaemonConfiguration);
+
+        assertThat(pollingConfig.kinesisClient(), equalTo(kinesisAsyncClient));
+        assertThat(pollingConfig.streamName(), equalTo(multiLangDaemonConfiguration.getStreamName()));
+        assertThat(
+                pollingConfig.idleTimeBetweenReadsInMillis(),
+                equalTo(pollingConfigBean.getIdleTimeBetweenReadsInMillis()));
+        assertThat(
+                pollingConfig.maxGetRecordsThreadPool(),
+                equalTo(Optional.of(pollingConfigBean.getMaxGetRecordsThreadPool())));
+        assertThat(pollingConfig.maxRecords(), equalTo(pollingConfigBean.getMaxRecords()));
+        assertThat(
+                pollingConfig.retryGetRecordsInSeconds(),
+                equalTo(Optional.of(pollingConfigBean.getRetryGetRecordsInSeconds())));
+    }
+}

amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/PropertiesMappingE2ETest.java

@@ -0,0 +1,303 @@
+package software.amazon.kinesis.multilang.config;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.Collections;
+
+import org.junit.jupiter.api.Test;
+import software.amazon.awssdk.services.dynamodb.model.BillingMode;
+import software.amazon.awssdk.services.dynamodb.model.Tag;
+import software.amazon.kinesis.coordinator.CoordinatorConfig.ClientVersionConfig;
+import software.amazon.kinesis.multilang.MultiLangDaemonConfig;
+import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration.ResolvedConfiguration;
+import software.amazon.kinesis.processor.ShardRecordProcessor;
+import software.amazon.kinesis.processor.ShardRecordProcessorFactory;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class PropertiesMappingE2ETest {
+    private static final String PROPERTIES_FILE = "multilang.properties";
+    private static final String PROPERTIES_FILE_V3 = "multilangv3.properties";
+
+    @Test
+    public void testKclV3PropertiesMapping() throws IOException {
+        final MultiLangDaemonConfig config = new MultiLangDaemonConfig(PROPERTIES_FILE);
+
+        final ResolvedConfiguration kclV3Config =
+                config.getMultiLangDaemonConfiguration().resolvedConfiguration(new TestRecordProcessorFactory());
+
+        assertEquals(
+                ClientVersionConfig.CLIENT_VERSION_CONFIG_COMPATIBLE_WITH_2X,
+                kclV3Config.coordinatorConfig.clientVersionConfig());
+
+        assertEquals(
+                "MultiLangTest-CoordinatorState-CustomName",
+                kclV3Config.coordinatorConfig.coordinatorStateTableConfig().tableName());
+        assertEquals(
+                BillingMode.PROVISIONED,
+                kclV3Config.coordinatorConfig.coordinatorStateTableConfig().billingMode());
+        assertEquals(
+                1000,
+                kclV3Config.coordinatorConfig.coordinatorStateTableConfig().readCapacity());
+        assertEquals(
+                500, kclV3Config.coordinatorConfig.coordinatorStateTableConfig().writeCapacity());
+        assertTrue(kclV3Config.coordinatorConfig.coordinatorStateTableConfig().pointInTimeRecoveryEnabled());
+        assertTrue(kclV3Config.coordinatorConfig.coordinatorStateTableConfig().deletionProtectionEnabled());
+        assertEquals(
+                Arrays.asList(
+                        Tag.builder().key("csTagK1").value("csTagV1").build(),
+                        Tag.builder().key("csTagK2").value("csTagV2").build(),
+                        Tag.builder().key("csTagK3").value("csTagV3").build()),
+                kclV3Config.coordinatorConfig.coordinatorStateTableConfig().tags());
+
+        assertEquals(
+                10000L,
+                kclV3Config.leaseManagementConfig.gracefulLeaseHandoffConfig().gracefulLeaseHandoffTimeoutMillis());
+        assertFalse(
+                kclV3Config.leaseManagementConfig.gracefulLeaseHandoffConfig().isGracefulLeaseHandoffEnabled());
+
+        assertEquals(
+                5000L,
+                kclV3Config
+                        .leaseManagementConfig
+                        .workerUtilizationAwareAssignmentConfig()
+                        .inMemoryWorkerMetricsCaptureFrequencyMillis());
+        assertEquals(
+                60000L,
+                kclV3Config
+                        .leaseManagementConfig
+                        .workerUtilizationAwareAssignmentConfig()
+                        .workerMetricsReporterFreqInMillis());
+        assertEquals(
+                50,
+                kclV3Config
+                        .leaseManagementConfig
+                        .workerUtilizationAwareAssignmentConfig()
+                        .noOfPersistedMetricsPerWorkerMetrics());
+        assertTrue(kclV3Config
+                .leaseManagementConfig
+                .workerUtilizationAwareAssignmentConfig()
+                .disableWorkerMetrics());
+        assertEquals(
+                10000,
+                kclV3Config
+                        .leaseManagementConfig
+                        .workerUtilizationAwareAssignmentConfig()
+                        .maxThroughputPerHostKBps());
+        assertEquals(
+                90,
+                kclV3Config
+                        .leaseManagementConfig
+                        .workerUtilizationAwareAssignmentConfig()
+                        .dampeningPercentage());
+        assertEquals(
+                5,
+                kclV3Config
+                        .leaseManagementConfig
+                        .workerUtilizationAwareAssignmentConfig()
+                        .reBalanceThresholdPercentage());
+        assertFalse(kclV3Config
+                .leaseManagementConfig
+                .workerUtilizationAwareAssignmentConfig()
+                .allowThroughputOvershoot());
+        assertEquals(
+                Duration.ofHours(12),
+                kclV3Config
+                        .leaseManagementConfig
+                        .workerUtilizationAwareAssignmentConfig()
+                        .staleWorkerMetricsEntryCleanupDuration());
+        assertEquals(
+                5,
+                kclV3Config
+                        .leaseManagementConfig
+                        .workerUtilizationAwareAssignmentConfig()
+                        .varianceBalancingFrequency());
+        assertEquals(
+                0.18D,
+                kclV3Config
+                        .leaseManagementConfig
+                        .workerUtilizationAwareAssignmentConfig()
+                        .workerMetricsEMAAlpha());
+
+        assertEquals(
+                "MultiLangTest-WorkerMetrics-CustomName",
+                kclV3Config
+                        .leaseManagementConfig
+                        .workerUtilizationAwareAssignmentConfig()
+                        .workerMetricsTableConfig()
+                        .tableName());
+        assertEquals(
+                BillingMode.PROVISIONED,
+                kclV3Config
+                        .leaseManagementConfig
+                        .workerUtilizationAwareAssignmentConfig()
+                        .workerMetricsTableConfig()
+                        .billingMode());
+        assertEquals(
+                250,
+                kclV3Config
+                        .leaseManagementConfig
+                        .workerUtilizationAwareAssignmentConfig()
+                        .workerMetricsTableConfig()
+                        .readCapacity());
+        assertEquals(
+                90,
+                kclV3Config
+                        .leaseManagementConfig
+                        .workerUtilizationAwareAssignmentConfig()
+                        .workerMetricsTableConfig()
+                        .writeCapacity());
+        assertTrue(kclV3Config
+                .leaseManagementConfig
+                .workerUtilizationAwareAssignmentConfig()
+                .workerMetricsTableConfig()
+                .pointInTimeRecoveryEnabled());
+        assertTrue(kclV3Config
+                .leaseManagementConfig
+                .workerUtilizationAwareAssignmentConfig()
+                .workerMetricsTableConfig()
+                .deletionProtectionEnabled());
+        assertEquals(
+                Arrays.asList(
+                        Tag.builder().key("wmTagK1").value("wmTagV1").build(),
+                        Tag.builder().key("wmTagK2").value("wmTagV2").build()),
+                kclV3Config
+                        .leaseManagementConfig
+                        .workerUtilizationAwareAssignmentConfig()
+                        .workerMetricsTableConfig()
+                        .tags());
+    }
+
+    @Test
+    public void testKclV3PropertiesMappingForDefaultValues() throws IOException {
+        final MultiLangDaemonConfig config = new MultiLangDaemonConfig(PROPERTIES_FILE_V3);
+
+        final ResolvedConfiguration kclV3Config =
+                config.getMultiLangDaemonConfiguration().resolvedConfiguration(new TestRecordProcessorFactory());
+
+        assertEquals(ClientVersionConfig.CLIENT_VERSION_CONFIG_3X, kclV3Config.coordinatorConfig.clientVersionConfig());
+
+        assertEquals(
+                "MultiLangTest-CoordinatorState",
+                kclV3Config.coordinatorConfig.coordinatorStateTableConfig().tableName());
+        assertEquals(
+                BillingMode.PAY_PER_REQUEST,
+                kclV3Config.coordinatorConfig.coordinatorStateTableConfig().billingMode());
+        assertFalse(kclV3Config.coordinatorConfig.coordinatorStateTableConfig().pointInTimeRecoveryEnabled());
+        assertFalse(kclV3Config.coordinatorConfig.coordinatorStateTableConfig().deletionProtectionEnabled());
+        assertEquals(
+                Collections.emptyList(),
+                kclV3Config.coordinatorConfig.coordinatorStateTableConfig().tags());
+
+        assertEquals(
+                30_000L,
+                kclV3Config.leaseManagementConfig.gracefulLeaseHandoffConfig().gracefulLeaseHandoffTimeoutMillis());
+        assertTrue(
+                kclV3Config.leaseManagementConfig.gracefulLeaseHandoffConfig().isGracefulLeaseHandoffEnabled());
+
+        assertEquals(
+                1000L,
+                kclV3Config
+                        .leaseManagementConfig
+                        .workerUtilizationAwareAssignmentConfig()
+                        .inMemoryWorkerMetricsCaptureFrequencyMillis());
+        assertEquals(
+                30000L,
+                kclV3Config
+                        .leaseManagementConfig
+                        .workerUtilizationAwareAssignmentConfig()
+                        .workerMetricsReporterFreqInMillis());
+        assertEquals(
+                10,
+                kclV3Config
+                        .leaseManagementConfig
+                        .workerUtilizationAwareAssignmentConfig()
+                        .noOfPersistedMetricsPerWorkerMetrics());
+        assertFalse(kclV3Config
+                .leaseManagementConfig
+                .workerUtilizationAwareAssignmentConfig()
+                .disableWorkerMetrics());
+        assertEquals(
+                Double.MAX_VALUE,
+                kclV3Config
+                        .leaseManagementConfig
+                        .workerUtilizationAwareAssignmentConfig()
+                        .maxThroughputPerHostKBps());
+        assertEquals(
+                60,
+                kclV3Config
+                        .leaseManagementConfig
+                        .workerUtilizationAwareAssignmentConfig()
+                        .dampeningPercentage());
+        assertEquals(
+                10,
+                kclV3Config
+                        .leaseManagementConfig
+                        .workerUtilizationAwareAssignmentConfig()
+                        .reBalanceThresholdPercentage());
+        assertTrue(kclV3Config
+                .leaseManagementConfig
+                .workerUtilizationAwareAssignmentConfig()
+                .allowThroughputOvershoot());
+        assertEquals(
+                Duration.ofDays(1),
+                kclV3Config
+                        .leaseManagementConfig
+                        .workerUtilizationAwareAssignmentConfig()
+                        .staleWorkerMetricsEntryCleanupDuration());
+        assertEquals(
+                3,
+                kclV3Config
+                        .leaseManagementConfig
+                        .workerUtilizationAwareAssignmentConfig()
+                        .varianceBalancingFrequency());
+        assertEquals(
+                0.5D,
+                kclV3Config
+                        .leaseManagementConfig
+                        .workerUtilizationAwareAssignmentConfig()
+                        .workerMetricsEMAAlpha());
+
+        assertEquals(
+                "MultiLangTest-WorkerMetricStats",
+                kclV3Config
+                        .leaseManagementConfig
+                        .workerUtilizationAwareAssignmentConfig()
+                        .workerMetricsTableConfig()
+                        .tableName());
+        assertEquals(
+                BillingMode.PAY_PER_REQUEST,
+                kclV3Config
+                        .leaseManagementConfig
+                        .workerUtilizationAwareAssignmentConfig()
+                        .workerMetricsTableConfig()
+                        .billingMode());
+        assertFalse(kclV3Config
+                .leaseManagementConfig
+                .workerUtilizationAwareAssignmentConfig()
+                .workerMetricsTableConfig()
+                .pointInTimeRecoveryEnabled());
+        assertFalse(kclV3Config
+                .leaseManagementConfig
+                .workerUtilizationAwareAssignmentConfig()
+                .workerMetricsTableConfig()
+                .deletionProtectionEnabled());
+        assertEquals(
+                Collections.emptyList(),
+                kclV3Config
+                        .leaseManagementConfig
+                        .workerUtilizationAwareAssignmentConfig()
+                        .workerMetricsTableConfig()
+                        .tags());
+    }
+
+    private static class TestRecordProcessorFactory implements ShardRecordProcessorFactory {
+        @Override
+        public ShardRecordProcessor shardRecordProcessor() {
+            return null;
+        }
+    }
+}

amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/WorkerUtilizationAwareAssignmentConfigBeanTest.java

@@ -0,0 +1,68 @@
+/*
+ * Copyright 2024 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.multilang.config;
+
+import java.util.Optional;
+
+import org.apache.commons.beanutils.BeanUtilsBean;
+import org.apache.commons.beanutils.ConvertUtilsBean;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.runners.MockitoJUnitRunner;
+import software.amazon.awssdk.services.kinesis.KinesisAsyncClient;
+import software.amazon.kinesis.retrieval.polling.PollingConfig;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.junit.Assert.assertThat;
+
+@RunWith(MockitoJUnitRunner.class)
+public class WorkerUtilizationAwareAssignmentConfigBeanTest {
+
+    @Mock
+    private KinesisAsyncClient kinesisAsyncClient;
+
+    @Test
+    public void testAllPropertiesTransit() {
+        PollingConfigBean pollingConfigBean = new PollingConfigBean();
+        pollingConfigBean.setIdleTimeBetweenReadsInMillis(1000);
+        pollingConfigBean.setMaxGetRecordsThreadPool(20);
+        pollingConfigBean.setMaxRecords(5000);
+        pollingConfigBean.setRetryGetRecordsInSeconds(30);
+
+        ConvertUtilsBean convertUtilsBean = new ConvertUtilsBean();
+        BeanUtilsBean utilsBean = new BeanUtilsBean(convertUtilsBean);
+
+        MultiLangDaemonConfiguration multiLangDaemonConfiguration =
+                new MultiLangDaemonConfiguration(utilsBean, convertUtilsBean);
+        multiLangDaemonConfiguration.setStreamName("test-stream");
+
+        PollingConfig pollingConfig = pollingConfigBean.build(kinesisAsyncClient, multiLangDaemonConfiguration);
+
+        assertThat(pollingConfig.kinesisClient(), equalTo(kinesisAsyncClient));
+        assertThat(pollingConfig.streamName(), equalTo(multiLangDaemonConfiguration.getStreamName()));
+        assertThat(
+                pollingConfig.idleTimeBetweenReadsInMillis(),
+                equalTo(pollingConfigBean.getIdleTimeBetweenReadsInMillis()));
+        assertThat(
+                pollingConfig.maxGetRecordsThreadPool(),
+                equalTo(Optional.of(pollingConfigBean.getMaxGetRecordsThreadPool())));
+        assertThat(pollingConfig.maxRecords(), equalTo(pollingConfigBean.getMaxRecords()));
+        assertThat(
+                pollingConfig.retryGetRecordsInSeconds(),
+                equalTo(Optional.of(pollingConfigBean.getRetryGetRecordsInSeconds())));
+    }
+}

amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/messages/JsonFriendlyRecordTest.java

@@ -0,0 +1,171 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.multilang.messages;
+
+import java.nio.ByteBuffer;
+import java.time.Instant;
+import java.util.Arrays;
+import java.util.List;
+import java.util.function.Function;
+import java.util.function.Supplier;
+
+import org.hamcrest.Description;
+import org.hamcrest.Matcher;
+import org.hamcrest.TypeSafeDiagnosingMatcher;
+import org.junit.Test;
+import software.amazon.kinesis.retrieval.KinesisClientRecord;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.nullValue;
+import static org.hamcrest.CoreMatchers.sameInstance;
+import static org.junit.Assert.assertThat;
+
+public class JsonFriendlyRecordTest {
+
+    private KinesisClientRecord kinesisClientRecord;
+
+    @Test
+    public void testRecordHandlesNullData() {
+        kinesisClientRecord = defaultRecord().data(null).build();
+        JsonFriendlyRecord jsonFriendlyRecord = JsonFriendlyRecord.fromKinesisClientRecord(kinesisClientRecord);
+
+        assertThat(jsonFriendlyRecord, equivalentTo(kinesisClientRecord));
+    }
+
+    @Test
+    public void testRecordHandlesNoByteArrayBuffer() {
+        byte[] expected = new byte[] {1, 2, 3, 4};
+
+        ByteBuffer expectedBuffer = ByteBuffer.allocateDirect(expected.length);
+
+        expectedBuffer.put(expected);
+        expectedBuffer.rewind();
+
+        kinesisClientRecord = defaultRecord().data(expectedBuffer).build();
+        JsonFriendlyRecord jsonFriendlyRecord = JsonFriendlyRecord.fromKinesisClientRecord(kinesisClientRecord);
+
+        expectedBuffer.rewind();
+        assertThat(jsonFriendlyRecord, equivalentTo(kinesisClientRecord));
+    }
+
+    @Test
+    public void testRecordHandlesArrayByteBuffer() {
+        ByteBuffer expected = ByteBuffer.wrap(new byte[] {1, 2, 3, 4});
+        kinesisClientRecord = defaultRecord().data(expected).build();
+        JsonFriendlyRecord jsonFriendlyRecord = JsonFriendlyRecord.fromKinesisClientRecord(kinesisClientRecord);
+
+        assertThat(jsonFriendlyRecord, equivalentTo(kinesisClientRecord));
+    }
+
+    private static RecordMatcher equivalentTo(KinesisClientRecord expected) {
+        return new RecordMatcher(expected);
+    }
+
+    private static class RecordMatcher extends TypeSafeDiagnosingMatcher<JsonFriendlyRecord> {
+
+        private final KinesisClientRecord expected;
+        private final List<Matcher<?>> matchers;
+
+        private RecordMatcher(KinesisClientRecord expected) {
+            this.matchers = Arrays.asList(
+                    new FieldMatcher<>(
+                            "approximateArrivalTimestamp",
+                            equalTo(expected.approximateArrivalTimestamp().toEpochMilli()),
+                            JsonFriendlyRecord::getApproximateArrivalTimestamp),
+                    new FieldMatcher<>("partitionKey", expected::partitionKey, JsonFriendlyRecord::getPartitionKey),
+                    new FieldMatcher<>(
+                            "sequenceNumber", expected::sequenceNumber, JsonFriendlyRecord::getSequenceNumber),
+                    new FieldMatcher<>(
+                            "subSequenceNumber", expected::subSequenceNumber, JsonFriendlyRecord::getSubSequenceNumber),
+                    new FieldMatcher<>("data", dataEquivalentTo(expected.data()), JsonFriendlyRecord::getData));
+
+            this.expected = expected;
+        }
+
+        @Override
+        protected boolean matchesSafely(JsonFriendlyRecord item, Description mismatchDescription) {
+            return matchers.stream()
+                    .map(m -> {
+                        if (!m.matches(item)) {
+                            m.describeMismatch(item, mismatchDescription);
+                            return false;
+                        }
+                        return true;
+                    })
+                    .reduce((l, r) -> l && r)
+                    .orElse(true);
+        }
+
+        @Override
+        public void describeTo(Description description) {
+            description.appendText("A JsonFriendlyRecord matching ").appendList("(", ", ", ")", matchers);
+        }
+    }
+
+    private static Matcher<Object> dataEquivalentTo(ByteBuffer expected) {
+        if (expected == null) {
+            return nullValue();
+        } else {
+            if (expected.hasArray()) {
+                return sameInstance(expected.array());
+            } else {
+                byte[] contents = new byte[expected.limit()];
+                expected.get(contents);
+                return equalTo(contents);
+            }
+        }
+    }
+
+    private static class FieldMatcher<ItemT, ClassT> extends TypeSafeDiagnosingMatcher<ClassT> {
+
+        final String fieldName;
+        final Matcher<ItemT> matcher;
+        final Function<ClassT, ItemT> extractor;
+
+        private FieldMatcher(String fieldName, Supplier<ItemT> expected, Function<ClassT, ItemT> extractor) {
+            this(fieldName, equalTo(expected.get()), extractor);
+        }
+
+        private FieldMatcher(String fieldName, Matcher<ItemT> matcher, Function<ClassT, ItemT> extractor) {
+            this.fieldName = fieldName;
+            this.matcher = matcher;
+            this.extractor = extractor;
+        }
+
+        @Override
+        protected boolean matchesSafely(ClassT item, Description mismatchDescription) {
+            ItemT actual = extractor.apply(item);
+            if (!matcher.matches(actual)) {
+                mismatchDescription.appendText(fieldName).appendText(": ");
+                matcher.describeMismatch(actual, mismatchDescription);
+                return false;
+            }
+            return true;
+        }
+
+        @Override
+        public void describeTo(Description description) {
+            description.appendText(fieldName).appendText(": ").appendDescriptionOf(matcher);
+        }
+    }
+
+    private KinesisClientRecord.KinesisClientRecordBuilder defaultRecord() {
+        return KinesisClientRecord.builder()
+                .partitionKey("test-partition")
+                .sequenceNumber("123")
+                .approximateArrivalTimestamp(Instant.now());
+    }
+}

amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/messages/MessageTest.java

@@ -0,0 +1,84 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package software.amazon.kinesis.multilang.messages;
+
+import java.nio.ByteBuffer;
+import java.util.Collections;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.Assert;
+import org.junit.Test;
+import software.amazon.kinesis.lifecycle.ShutdownReason;
+import software.amazon.kinesis.lifecycle.events.InitializationInput;
+import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput;
+import software.amazon.kinesis.retrieval.KinesisClientRecord;
+
+public class MessageTest {
+
+    @Test
+    public void toStringTest() {
+        Message[] messages = new Message[] {
+            new CheckpointMessage("1234567890", 0L, null),
+            new InitializeMessage(
+                    InitializationInput.builder().shardId("shard-123").build()),
+            new ProcessRecordsMessage(ProcessRecordsInput.builder()
+                    .records(Collections.singletonList(KinesisClientRecord.builder()
+                            .data(ByteBuffer.wrap("cat".getBytes()))
+                            .partitionKey("cat")
+                            .sequenceNumber("555")
+                            .build()))
+                    .build()),
+            new ShutdownMessage(ShutdownReason.LEASE_LOST),
+            new StatusMessage("processRecords"),
+            new InitializeMessage(),
+            new ProcessRecordsMessage(),
+            new ShutdownRequestedMessage(),
+            new LeaseLostMessage(),
+            new ShardEndedMessage(),
+        };
+
+        //        TODO: fix this
+        for (int i = 0; i < messages.length; i++) {
+            System.out.println(messages[i].toString());
+            Assert.assertTrue(
+                    "Each message should contain the action field",
+                    messages[i].toString().contains("action"));
+        }
+
+        // Hit this constructor
+        KinesisClientRecord defaultJsonFriendlyRecord =
+                KinesisClientRecord.builder().build();
+        Assert.assertNull(defaultJsonFriendlyRecord.partitionKey());
+        Assert.assertNull(defaultJsonFriendlyRecord.data());
+        Assert.assertNull(defaultJsonFriendlyRecord.sequenceNumber());
+        Assert.assertNull(new ShutdownMessage(null).getReason());
+
+        // Hit the bad object mapping path
+        Message withBadMapper = new Message() {}.withObjectMapper(new ObjectMapper() {
+            /**
+             *
+             */
+            private static final long serialVersionUID = 1L;
+
+            @Override
+            public String writeValueAsString(Object m) throws JsonProcessingException {
+                throw new JsonProcessingException(new Throwable()) {};
+            }
+        });
+        String s = withBadMapper.toString();
+        Assert.assertNotNull(s);
+    }
+}

amazon-kinesis-client-multilang/src/test/resources/logback-test.xml

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+  -->
+<configuration>
+    <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <pattern>%d [%thread] %-5level %logger{36} [%mdc{ShardId:-NONE}] - %msg %n</pattern>
+        </encoder>
+    </appender>
+
+    <root level="INFO">
+        <appender-ref ref="CONSOLE" />
+    </root>
+</configuration>

amazon-kinesis-client-multilang/src/test/resources/multilang.properties

@@ -0,0 +1,175 @@
+# The script that abides by the multi-language protocol. This script will
+# be executed by the MultiLangDaemon, which will communicate with this script
+# over STDIN and STDOUT according to the multi-language protocol.
+executableName = sample_kclpy_app.py
+
+# The Stream arn: arn:aws:kinesis:<region>:<account id>:stream/<stream name>
+# Important: streamArn takes precedence over streamName if both are set
+streamArn = arn:aws:kinesis:us-east-5:000000000000:stream/kclpysample
+
+# The name of an Amazon Kinesis stream to process.
+# Important: streamArn takes precedence over streamName if both are set
+streamName = kclpysample
+
+# Used by the KCL as the name of this application. Will be used as the name
+# of an Amazon DynamoDB table which will store the lease and checkpoint
+# information for workers with this application name
+applicationName = MultiLangTest
+
+# Users can change the credentials provider the KCL will use to retrieve credentials.
+# Expected key name (case-sensitive):
+# AwsCredentialsProvider / AwsCredentialsProviderDynamoDB / AwsCredentialsProviderCloudWatch
+# The DefaultCredentialsProvider checks several other providers, which is
+# described here:
+# https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/auth/credentials/DefaultCredentialsProvider.html
+AwsCredentialsProvider = DefaultCredentialsProvider
+
+# Appended to the user agent of the KCL. Does not impact the functionality of the
+# KCL in any other way.
+processingLanguage = python/3.8
+
+# Valid options at TRIM_HORIZON or LATEST.
+# See http://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetShardIterator.html#API_GetShardIterator_RequestSyntax
+initialPositionInStream = TRIM_HORIZON
+
+# To specify an initial timestamp from which to start processing records, please specify timestamp value for 'initiatPositionInStreamExtended',
+# and uncomment below line with right timestamp value.
+# See more from 'Timestamp' under http://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetShardIterator.html#API_GetShardIterator_RequestSyntax
+#initialPositionInStreamExtended = 1636609142
+
+# The following properties are also available for configuring the KCL Worker that is created
+# by the MultiLangDaemon.
+
+# The KCL defaults to us-east-1
+regionName = us-east-1
+
+# Fail over time in milliseconds. A worker which does not renew it's lease within this time interval
+# will be regarded as having problems and it's shards will be assigned to other workers.
+# For applications that have a large number of shards, this msy be set to a higher number to reduce
+# the number of DynamoDB IOPS required for tracking leases
+failoverTimeMillis = 10000
+
+# A worker id that uniquely identifies this worker among all workers using the same applicationName
+# If this isn't provided a MultiLangDaemon instance will assign a unique workerId to itself.
+workerId = "workerId"
+
+# Shard sync interval in milliseconds - e.g. wait for this long between shard sync tasks.
+shardSyncIntervalMillis = 60000
+
+# Max records to fetch from Kinesis in a single GetRecords call.
+maxRecords = 10000
+
+# Idle time between record reads in milliseconds.
+idleTimeBetweenReadsInMillis = 1000
+
+# Enables applications flush/checkpoint (if they have some data "in progress", but don't get new data for while)
+callProcessRecordsEvenForEmptyRecordList = false
+
+# Interval in milliseconds between polling to check for parent shard completion.
+# Polling frequently will take up more DynamoDB IOPS (when there are leases for shards waiting on
+# completion of parent shards).
+parentShardPollIntervalMillis = 10000
+
+# Cleanup leases upon shards completion (don't wait until they expire in Kinesis).
+# Keeping leases takes some tracking/resources (e.g. they need to be renewed, assigned), so by default we try
+# to delete the ones we don't need any longer.
+cleanupLeasesUponShardCompletion = true
+
+# Backoff time in milliseconds for Amazon Kinesis Client Library tasks (in the event of failures).
+taskBackoffTimeMillis = 500
+
+# Buffer metrics for at most this long before publishing to CloudWatch.
+metricsBufferTimeMillis = 10000
+
+# Buffer at most this many metrics before publishing to CloudWatch.
+metricsMaxQueueSize = 10000
+
+# KCL will validate client provided sequence numbers with a call to Amazon Kinesis before checkpointing for calls
+# to RecordProcessorCheckpointer#checkpoint(String) by default.
+validateSequenceNumberBeforeCheckpointing = true
+
+# The maximum number of active threads for the MultiLangDaemon to permit.
+# If a value is provided then a FixedThreadPool is used with the maximum
+# active threads set to the provided value. If a non-positive integer or no
+# value is provided a CachedThreadPool is used.
+maxActiveThreads = -1
+
+################### KclV3 configurations ###################
+# NOTE : These are just test configurations to show how to customize
+# all possible KCLv3 configurations. They are not necessarily the best
+# default values to use for production.
+
+# Coordinator config
+# Version the KCL needs to operate in. For more details check the KCLv3 migration
+# documentation. Default is CLIENT_VERSION_CONFIG_3X
+clientVersionConfig = CLIENT_VERSION_CONFIG_COMPATIBLE_WITH_2x
+# Configurations to control how the CoordinatorState DDB table is created
+# Default name is applicationName-CoordinatorState in PAY_PER_REQUEST,
+# with PITR and deletion protection disabled and no tags
+coordinatorStateTableName = MultiLangTest-CoordinatorState-CustomName
+coordinatorStateBillingMode = PROVISIONED
+coordinatorStateReadCapacity = 1000
+coordinatorStateWriteCapacity = 500
+coordinatorStatePointInTimeRecoveryEnabled = true
+coordinatorStateDeletionProtectionEnabled = true
+coordinatorStateTags = csTagK1=csTagV1,csTagK2=csTagV2,csTagK3=csTagV3
+
+# Graceful handoff config - tuning of the shutdown behavior during lease transfers
+# default values are 30000 and true respectively
+gracefulLeaseHandoffTimeoutMillis = 10000
+isGracefulLeaseHandoffEnabled = false
+
+# WorkerMetricStats table config - control how the DDB table is created
+# Default name is applicationName-WorkerMetricStats in PAY_PER_REQUEST,
+# with PITR and deletion protection disabled and no tags
+workerMetricsTableName = MultiLangTest-WorkerMetrics-CustomName
+workerMetricsBillingMode = PROVISIONED
+workerMetricsReadCapacity = 250
+workerMetricsWriteCapacity = 90
+workerMetricsPointInTimeRecoveryEnabled = true
+workerMetricsDeletionProtectionEnabled = true
+workerMetricsTags = wmTagK1=wmTagV1,wmTagK2=wmTagV2
+
+# WorkerUtilizationAwareAssignment config - tune the new KCLv3 Lease balancing algorithm
+#
+# frequency of capturing worker metrics in memory. Default is 1s
+inMemoryWorkerMetricsCaptureFrequencyMillis = 5000
+# frequency of reporting worker metric stats to storage. Default is 30s
+workerMetricsReporterFreqInMillis = 60000
+# No. of metricStats that are persisted in WorkerMetricStats ddb table, default is 10
+noOfPersistedMetricsPerWorkerMetrics = 50
+# Disable use of worker metrics to balance lease, default is false.
+# If it is true, the algorithm balances lease based on worker's processing throughput.
+disableWorkerMetrics = true
+# Max throughput per host 10 MBps, to limit processing to the given value
+# Default is unlimited.
+maxThroughputPerHostKBps = 10000
+# Dampen the load that is rebalanced during lease re-balancing, default is 60%
+dampeningPercentage = 90
+# Configures the allowed variance range for worker utilization. The upper
+# limit is calculated as average * (1 + reBalanceThresholdPercentage/100).
+# The lower limit is average * (1 - reBalanceThresholdPercentage/100). If
+# any worker's utilization falls outside this range, lease re-balancing is
+# triggered. The re-balancing algorithm aims to bring variance within the
+# specified range. It also avoids thrashing by ensuring the utilization of
+# the worker receiving the load after re-balancing doesn't exceed the fleet
+# average. This might cause no re-balancing action even the utilization is
+# out of the variance range. The default value is 10, representing +/-10%
+# variance from the average value.
+reBalanceThresholdPercentage = 5
+# Whether at-least one lease must be taken from a high utilization worker
+# during re-balancing when there is no lease assigned to that worker which has
+# throughput is less than or equal to the minimum throughput that needs to be
+# moved away from that worker to bring the worker back into the allowed variance.
+# Default is true.
+allowThroughputOvershoot = false
+# Lease assignment is performed every failoverTimeMillis but re-balance will
+# be attempted only once in 5 times based on the below config. Default is 3.
+varianceBalancingFrequency = 5
+# Alpha value used for calculating exponential moving average of worker's metricStats.
+workerMetricsEMAAlpha = 0.18
+# Duration after which workerMetricStats entry from WorkerMetricStats table will
+# be cleaned up.
+# Duration format examples: PT15M (15 mins) PT10H (10 hours) P2D (2 days)
+# Refer to Duration.parse javadocs for more details
+staleWorkerMetricsEntryCleanupDuration = PT12H

amazon-kinesis-client-multilang/src/test/resources/multilangv3.properties

@@ -0,0 +1,100 @@
+# The script that abides by the multi-language protocol. This script will
+# be executed by the MultiLangDaemon, which will communicate with this script
+# over STDIN and STDOUT according to the multi-language protocol.
+executableName = sample_kclpy_app.py
+
+# The Stream arn: arn:aws:kinesis:<region>:<account id>:stream/<stream name>
+# Important: streamArn takes precedence over streamName if both are set
+streamArn = arn:aws:kinesis:us-east-5:000000000000:stream/kclpysample
+
+# The name of an Amazon Kinesis stream to process.
+# Important: streamArn takes precedence over streamName if both are set
+streamName = kclpysample
+
+# Used by the KCL as the name of this application. Will be used as the name
+# of an Amazon DynamoDB table which will store the lease and checkpoint
+# information for workers with this application name
+applicationName = MultiLangTest
+
+# Users can change the credentials provider the KCL will use to retrieve credentials.
+# Expected key name (case-sensitive):
+# AwsCredentialsProvider / AwsCredentialsProviderDynamoDB / AwsCredentialsProviderCloudWatch
+# The DefaultCredentialsProvider checks several other providers, which is
+# described here:
+# https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/auth/credentials/DefaultCredentialsProvider.html
+AwsCredentialsProvider = DefaultCredentialsProvider
+
+# Appended to the user agent of the KCL. Does not impact the functionality of the
+# KCL in any other way.
+processingLanguage = python/3.8
+
+# Valid options at TRIM_HORIZON or LATEST.
+# See http://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetShardIterator.html#API_GetShardIterator_RequestSyntax
+initialPositionInStream = TRIM_HORIZON
+
+# To specify an initial timestamp from which to start processing records, please specify timestamp value for 'initiatPositionInStreamExtended',
+# and uncomment below line with right timestamp value.
+# See more from 'Timestamp' under http://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetShardIterator.html#API_GetShardIterator_RequestSyntax
+#initialPositionInStreamExtended = 1636609142
+
+# The following properties are also available for configuring the KCL Worker that is created
+# by the MultiLangDaemon.
+
+# The KCL defaults to us-east-1
+regionName = us-east-1
+
+# Fail over time in milliseconds. A worker which does not renew it's lease within this time interval
+# will be regarded as having problems and it's shards will be assigned to other workers.
+# For applications that have a large number of shards, this msy be set to a higher number to reduce
+# the number of DynamoDB IOPS required for tracking leases
+failoverTimeMillis = 10000
+
+# A worker id that uniquely identifies this worker among all workers using the same applicationName
+# If this isn't provided a MultiLangDaemon instance will assign a unique workerId to itself.
+workerId = "workerId"
+
+# Shard sync interval in milliseconds - e.g. wait for this long between shard sync tasks.
+shardSyncIntervalMillis = 60000
+
+# Max records to fetch from Kinesis in a single GetRecords call.
+maxRecords = 10000
+
+# Idle time between record reads in milliseconds.
+idleTimeBetweenReadsInMillis = 1000
+
+# Enables applications flush/checkpoint (if they have some data "in progress", but don't get new data for while)
+callProcessRecordsEvenForEmptyRecordList = false
+
+# Interval in milliseconds between polling to check for parent shard completion.
+# Polling frequently will take up more DynamoDB IOPS (when there are leases for shards waiting on
+# completion of parent shards).
+parentShardPollIntervalMillis = 10000
+
+# Cleanup leases upon shards completion (don't wait until they expire in Kinesis).
+# Keeping leases takes some tracking/resources (e.g. they need to be renewed, assigned), so by default we try
+# to delete the ones we don't need any longer.
+cleanupLeasesUponShardCompletion = true
+
+# Backoff time in milliseconds for Amazon Kinesis Client Library tasks (in the event of failures).
+taskBackoffTimeMillis = 500
+
+# Buffer metrics for at most this long before publishing to CloudWatch.
+metricsBufferTimeMillis = 10000
+
+# Buffer at most this many metrics before publishing to CloudWatch.
+metricsMaxQueueSize = 10000
+
+# KCL will validate client provided sequence numbers with a call to Amazon Kinesis before checkpointing for calls
+# to RecordProcessorCheckpointer#checkpoint(String) by default.
+validateSequenceNumberBeforeCheckpointing = true
+
+# The maximum number of active threads for the MultiLangDaemon to permit.
+# If a value is provided then a FixedThreadPool is used with the maximum
+# active threads set to the provided value. If a non-positive integer or no
+# value is provided a CachedThreadPool is used.
+maxActiveThreads = -1
+
+################### KclV3 configurations ###################
+# Coordinator config
+clientVersionConfig = CLIENT_VERSION_CONFIG_3x
+## Let all other KCLv3 config use defaults

amazon-kinesis-client/pom.xml

@@ -0,0 +1,591 @@
+<!--
+
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+  -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <groupId>software.amazon.kinesis</groupId>
+    <artifactId>amazon-kinesis-client-pom</artifactId>
+    <version>3.0.3</version>
+  </parent>
+
+  <artifactId>amazon-kinesis-client</artifactId>
+  <packaging>jar</packaging>
+  <name>Amazon Kinesis Client Library for Java</name>
+
+  <description>The Amazon Kinesis Client Library for Java enables Java developers to easily consume and process data
+    from Amazon Kinesis.
+  </description>
+  <url>https://aws.amazon.com/kinesis</url>
+
+  <scm>
+    <url>https://github.com/awslabs/amazon-kinesis-client.git</url>
+  </scm>
+
+  <licenses>
+    <license>
+      <name>Apache License, Version 2.0</name>
+      <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+      <distribution>repo</distribution>
+    </license>
+  </licenses>
+
+  <properties>
+    <protobuf.version>4.27.5</protobuf.version>
+    <sqlite4java.version>1.0.392</sqlite4java.version>
+    <sqlite4java.native>libsqlite4java</sqlite4java.native>
+    <sqlite4java.libpath>${project.build.directory}/test-lib</sqlite4java.libpath>
+    <slf4j.version>2.0.13</slf4j.version>
+    <gsr.version>1.1.19</gsr.version>
+    <skipITs>true</skipITs>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>kinesis</artifactId>
+      <version>${awssdk.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>dynamodb</artifactId>
+      <version>${awssdk.version}</version>
+    </dependency>
+    <!-- https://mvnrepository.com/artifact/software.amazon.awssdk/dynamodb-enhanced -->
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>dynamodb-enhanced</artifactId>
+      <version>${awssdk.version}</version>
+    </dependency>
+    <!-- https://mvnrepository.com/artifact/com.amazonaws/dynamodb-lock-client -->
+    <dependency>
+      <groupId>com.amazonaws</groupId>
+      <artifactId>dynamodb-lock-client</artifactId>
+      <version>1.3.0</version>
+    </dependency>
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>cloudwatch</artifactId>
+      <version>${awssdk.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>netty-nio-client</artifactId>
+      <version>${awssdk.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>sdk-core</artifactId>
+      <version>${awssdk.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>aws-core</artifactId>
+      <version>${awssdk.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>arns</artifactId>
+      <version>${awssdk.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>regions</artifactId>
+      <version>${awssdk.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>utils</artifactId>
+      <version>${awssdk.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>http-client-spi</artifactId>
+      <version>${awssdk.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>dynamodb-enhanced</artifactId>
+      <version>${awssdk.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>software.amazon.glue</groupId>
+      <artifactId>schema-registry-serde</artifactId>
+      <version>${gsr.version}</version>
+      <exclusions>
+          <exclusion>
+              <groupId>com.amazonaws</groupId>
+              <artifactId>aws-java-sdk-sts</artifactId>
+          </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>software.amazon.glue</groupId>
+      <artifactId>schema-registry-common</artifactId>
+      <version>${gsr.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+      <version>32.1.1-jre</version>
+    </dependency>
+    <dependency>
+      <groupId>com.google.protobuf</groupId>
+      <artifactId>protobuf-java</artifactId>
+      <version>${protobuf.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-lang3</artifactId>
+      <version>3.14.0</version>
+    </dependency>
+    <!-- https://mvnrepository.com/artifact/commons-collections/commons-collections -->
+    <dependency>
+      <groupId>commons-collections</groupId>
+      <artifactId>commons-collections</artifactId>
+      <version>3.2.2</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-collections4</artifactId>
+      <version>4.4</version>
+    </dependency>
+    <dependency>
+      <groupId>io.netty</groupId>
+      <artifactId>netty-handler</artifactId>
+      <version>4.1.118.Final</version>
+    </dependency>
+    <dependency>
+      <groupId>com.google.code.findbugs</groupId>
+      <artifactId>jsr305</artifactId>
+      <version>3.0.2</version>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-databind</artifactId>
+      <version>2.12.7.1</version>
+    </dependency>
+    <dependency>
+      <groupId>org.reactivestreams</groupId>
+      <artifactId>reactive-streams</artifactId>
+      <version>1.0.4</version>
+    </dependency>
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>annotations</artifactId>
+      <version>2.25.64</version>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+      <version>${slf4j.version}</version>
+    </dependency>
+    <!-- https://mvnrepository.com/artifact/org.jetbrains/annotations -->
+    <dependency>
+      <groupId>org.jetbrains</groupId>
+      <artifactId>annotations</artifactId>
+      <version>26.0.1</version>
+    </dependency>
+
+    <dependency>
+      <groupId>io.reactivex.rxjava3</groupId>
+      <artifactId>rxjava</artifactId>
+      <version>3.1.8</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.projectlombok</groupId>
+      <artifactId>lombok</artifactId>
+      <version>1.18.24</version>
+      <scope>provided</scope>
+    </dependency>
+
+    <!-- Test -->
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>sts</artifactId>
+      <version>${awssdk.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>auth</artifactId>
+      <version>${awssdk.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <!-- TODO: Migrate all tests to Junit5    -->
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-api</artifactId>
+      <version>5.11.3</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>4.13.2</version>
+      <scope>test</scope>
+    </dependency>
+    <!-- https://mvnrepository.com/artifact/org.junit.jupiter/junit-jupiter-params -->
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-params</artifactId>
+      <version>5.11.3</version>
+      <scope>test</scope>
+    </dependency>
+    <!-- Using older version to be compatible with Java 8   -->
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-junit-jupiter</artifactId>
+      <version>3.12.4</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-core</artifactId>
+      <version>3.12.4</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.hamcrest</groupId>
+      <artifactId>hamcrest-all</artifactId>
+      <version>1.3</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.hamcrest</groupId>
+      <artifactId>hamcrest-core</artifactId>
+      <version>1.3</version>
+      <scope>test</scope>
+    </dependency>
+    <!-- Using older version to be compatible with Java 8   -->
+    <!-- https://mvnrepository.com/artifact/com.amazonaws/DynamoDBLocal -->
+    <dependency>
+      <groupId>com.amazonaws</groupId>
+      <artifactId>DynamoDBLocal</artifactId>
+      <version>1.25.0</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>ch.qos.logback</groupId>
+      <artifactId>logback-classic</artifactId>
+      <version>1.3.14</version>
+      <scope>test</scope>
+    </dependency>
+
+  </dependencies>
+
+  <!--<repositories>-->
+  <!--<repository>-->
+  <!--<id>dynamodblocal</id>-->
+  <!--<name>AWS DynamoDB Local Release Repository</name>-->
+  <!--<url>https://s3-us-west-2.amazonaws.com/dynamodb-local/release</url>-->
+  <!--</repository>-->
+  <!--</repositories>-->
+
+  <developers>
+    <developer>
+      <id>amazonwebservices</id>
+      <organization>Amazon Web Services</organization>
+      <organizationUrl>https://aws.amazon.com</organizationUrl>
+      <roles>
+        <role>developer</role>
+      </roles>
+    </developer>
+  </developers>
+
+  <build>
+    <extensions>
+      <extension>
+        <groupId>kr.motd.maven</groupId>
+        <artifactId>os-maven-plugin</artifactId>
+        <version>1.6.0</version>
+      </extension>
+    </extensions>
+    <pluginManagement>
+      <plugins>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-compiler-plugin</artifactId>
+          <version>3.13.0</version>
+          <configuration>
+            <release>8</release>
+            <encoding>UTF-8</encoding>
+          </configuration>
+        </plugin>
+      </plugins>
+    </pluginManagement>
+
+    <plugins>
+      <plugin>
+        <groupId>org.xolstice.maven.plugins</groupId>
+        <artifactId>protobuf-maven-plugin</artifactId>
+        <version>0.6.1</version>
+        <executions>
+          <execution>
+            <goals>
+              <goal>compile</goal>
+            </goals>
+          </execution>
+        </executions>
+        <configuration>
+          <protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <version>3.2.5</version>
+        <configuration>
+          <skipTests>${skip.ut}</skipTests>
+          <skipITs>${skipITs}</skipITs>
+          <excludes>
+            <exclude>**/*IntegrationTest.java</exclude>
+          </excludes>
+          <systemPropertyVariables>
+            <sqlite4java.library.path>${sqlite4java.libpath}</sqlite4java.library.path>
+            <awsProfile>${awsProfile}</awsProfile>
+          </systemPropertyVariables>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-failsafe-plugin</artifactId>
+        <version>3.2.5</version>
+        <configuration>
+          <includes>
+            <include>**/*IntegrationTest.java</include>
+          </includes>
+        </configuration>
+        <executions>
+          <execution>
+            <goals>
+              <goal>integration-test</goal>
+              <goal>verify</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>copy</id>
+            <phase>test-compile</phase>
+            <goals>
+              <goal>copy</goal>
+            </goals>
+            <configuration>
+              <artifactItems>
+                <!-- Mac OS X -->
+                <artifactItem>
+                  <groupId>com.almworks.sqlite4java</groupId>
+                  <artifactId>${sqlite4java.native}-osx</artifactId>
+                  <version>${sqlite4java.version}</version>
+                  <type>dylib</type>
+                  <overWrite>true</overWrite>
+                  <outputDirectory>${sqlite4java.libpath}</outputDirectory>
+                </artifactItem>
+
+                <!-- Linux -->
+                <!-- i386 -->
+                <artifactItem>
+                  <groupId>com.almworks.sqlite4java</groupId>
+                  <artifactId>${sqlite4java.native}-linux-i386</artifactId>
+                  <version>${sqlite4java.version}</version>
+                  <type>so</type>
+                  <overWrite>true</overWrite>
+                  <outputDirectory>${sqlite4java.libpath}</outputDirectory>
+                </artifactItem>
+
+                <!-- amd64 -->
+                <artifactItem>
+                  <groupId>com.almworks.sqlite4java</groupId>
+                  <artifactId>${sqlite4java.native}-linux-amd64</artifactId>
+                  <version>${sqlite4java.version}</version>
+                  <type>so</type>
+                  <overWrite>true</overWrite>
+                  <outputDirectory>${sqlite4java.libpath}</outputDirectory>
+                </artifactItem>
+
+                <!-- Windows -->
+                <!-- x86 -->
+                <artifactItem>
+                  <groupId>com.almworks.sqlite4java</groupId>
+                  <artifactId>sqlite4java-win32-x86</artifactId>
+                  <version>${sqlite4java.version}</version>
+                  <type>dll</type>
+                  <overWrite>true</overWrite>
+                  <outputDirectory>${sqlite4java.libpath}</outputDirectory>
+                </artifactItem>
+
+                <!-- x64 -->
+                <artifactItem>
+                  <groupId>com.almworks.sqlite4java</groupId>
+                  <artifactId>sqlite4java-win32-x64</artifactId>
+                  <version>${sqlite4java.version}</version>
+                  <type>dll</type>
+                  <overWrite>true</overWrite>
+                  <outputDirectory>${sqlite4java.libpath}</outputDirectory>
+                </artifactItem>
+              </artifactItems>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-javadoc-plugin</artifactId>
+        <version>3.7.0</version>
+        <configuration>
+          <excludePackageNames>com.amazonaws.services.kinesis.producer.protobuf</excludePackageNames>
+        </configuration>
+        <executions>
+          <execution>
+            <id>attach-javadocs</id>
+            <goals>
+              <goal>jar</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-source-plugin</artifactId>
+        <version>3.2.1</version>
+        <executions>
+          <execution>
+            <id>attach-sources</id>
+            <goals>
+              <goal>jar</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+
+      <!-- Required for generating maven version as a Java class for runtime access -->
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>templating-maven-plugin</artifactId>
+        <version>1.0.0</version>
+        <executions>
+          <execution>
+            <id>generate-version-class</id>
+            <goals>
+              <goal>filter-sources</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-resources-plugin</artifactId>
+        <version>3.3.1</version>
+        <executions>
+          <execution>
+            <id>copy-dist</id>
+            <phase>prepare-package</phase>
+            <goals>
+              <goal>copy-resources</goal>
+            </goals>
+            <configuration>
+              <outputDirectory>${project.build.outputDirectory}</outputDirectory>
+              <resources>
+                <resource>
+                  <directory>${project.basedir}/target/generated-sources/java-templates/</directory>
+                  <filtering>false</filtering>
+                  <excludes>
+                  </excludes>
+                </resource>
+              </resources>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>com.diffplug.spotless</groupId>
+        <artifactId>spotless-maven-plugin</artifactId>
+        <version>2.30.0</version> <!--last version to support java 8-->
+        <configuration>
+          <java>
+            <palantirJavaFormat />
+            <importOrder>
+              <order>java,,\#</order>
+            </importOrder>
+          </java>
+        </configuration>
+        <executions>
+          <execution>
+            <goals>
+              <goal>check</goal>
+            </goals>
+            <phase>compile</phase>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+         <groupId>com.salesforce.servicelibs</groupId>
+         <artifactId>proto-backwards-compatibility</artifactId>
+         <version>1.0.7</version>
+         <executions>
+             <execution>
+                 <goals>
+                     <goal>backwards-compatibility-check</goal>
+                 </goals>
+             </execution>
+         </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <version>3.1.2</version>
+        <executions>
+          <execution>
+            <id>analyze-dependencies</id>
+            <phase>verify</phase>
+            <goals>
+              <goal>analyze-only</goal>
+            </goals>
+            <configuration>
+              <failOnWarning>true</failOnWarning>
+              <!-- Ignore Runtime/Provided/Test/System scopes for unused dependency analysis. -->
+              <ignoreNonCompile>true</ignoreNonCompile>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+
+  </build>
+  <profiles>
+    <profile>
+      <id>disable-java8-doclint</id>
+      <activation>
+        <jdk>[1.8,)</jdk>
+      </activation>
+      <properties>
+        <doclint>none</doclint>
+      </properties>
+    </profile>
+  </profiles>
+
+</project>

amazon-kinesis-client/scripts/KclMigrationTool.py

@@ -0,0 +1,609 @@
+"""
+Copyright 2024 Amazon.com, Inc. or its affiliates.
+Licensed under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import argparse
+import time
+
+from enum import Enum
+import boto3
+from botocore.config import Config
+from botocore.exceptions import ClientError
+
+# DynamoDB table suffixes
+DEFAULT_COORDINATOR_STATE_TABLE_SUFFIX = "-CoordinatorState"
+DEFAULT_WORKER_METRICS_TABLE_SUFFIX = "-WorkerMetricStats"
+
+# DynamoDB attribute names and values
+CLIENT_VERSION_ATTR = 'cv'
+TIMESTAMP_ATTR = 'mts'
+MODIFIED_BY_ATTR = 'mb'
+HISTORY_ATTR = 'h'
+MIGRATION_KEY = "Migration3.0"
+
+# GSI constants
+GSI_NAME = 'LeaseOwnerToLeaseKeyIndex'
+GSI_DELETION_WAIT_TIME_SECONDS = 120
+
+config = Config(
+    retries = {
+        'max_attempts': 10,
+        'mode': 'standard'
+    }
+)
+
+class KclClientVersion(Enum):
+    VERSION_2X = "CLIENT_VERSION_2X"
+    UPGRADE_FROM_2X = "CLIENT_VERSION_UPGRADE_FROM_2X"
+    VERSION_3X_WITH_ROLLBACK = "CLIENT_VERSION_3X_WITH_ROLLBACK"
+    VERSION_3X = "CLIENT_VERSION_3X"
+
+    def __str__(self):
+        return self.value
+
+
+def get_time_in_millis():
+    return str(round(time.time() * 1000))
+
+
+def is_valid_version(version, mode):
+    """
+    Validate if the given version is valid for the specified mode
+
+    :param version: The KCL client version to validate
+    :param mode: Either 'rollback' or 'rollforward'
+    :return: True if the version is valid for the given mode, False otherwise
+    """
+    if mode == 'rollback':
+        if version == KclClientVersion.VERSION_2X.value:
+            print("Your KCL application already runs in a mode compatible with KCL 2.x. You can deploy the code with the previous KCL version if you still experience an issue.")
+            return True
+        if version in [KclClientVersion.UPGRADE_FROM_2X.value,
+                       KclClientVersion.VERSION_3X_WITH_ROLLBACK.value]:
+            return True
+        if version == KclClientVersion.VERSION_3X.value:
+            print("Cannot roll back the KCL application."
+                  " It is not in a state that supports rollback.")
+            return False
+        print("Migration to KCL 3.0 not in progress or application_name / coordinator_state_table_name is incorrect."
+              " Please double check and run again with correct arguments.")
+        return False
+
+    if mode == 'rollforward':
+        if version == KclClientVersion.VERSION_2X.value:
+            return True
+        if version in [KclClientVersion.UPGRADE_FROM_2X.value,
+                       KclClientVersion.VERSION_3X_WITH_ROLLBACK.value]:
+            print("Cannot roll-forward application. It is not in a rolled back state.")
+            return False
+        if version == KclClientVersion.VERSION_3X.value:
+            print("Cannot roll-forward the KCL application."
+                  " Application has already migrated.")
+            return False
+        print("Cannot roll-forward because migration to KCL 3.0 is not in progress or application_name"
+              " / coordinator_state_table_name is incorrect. Please double check and run again with correct arguments.")
+        return False
+    print(f"Invalid mode: {mode}. Mode must be either 'rollback' or 'rollforward'.")
+    return False
+
+
+def handle_get_item_client_error(e, operation, table_name):
+    """
+    Handle ClientError exceptions raised by get_item on given DynamoDB table
+
+    :param e: The ClientError exception object
+    :param operation: Rollback or Roll-forward for logging the errors
+    :param table_name: The name of the DynamoDB table where the error occurred
+    """
+    error_code = e.response['Error']['Code']
+    error_message = e.response['Error']['Message']
+    print(f"{operation} could not be performed.")
+    if error_code == 'ProvisionedThroughputExceededException':
+        print(f"Throughput exceeded even after retries: {error_message}")
+    else:
+        print(f"Unexpected client error occurred: {error_code} - {error_message}")
+    print("Please resolve the issue and run the KclMigrationTool again.")
+
+
+def table_exists(dynamodb_client, table_name):
+    """
+    Check if a DynamoDB table exists.
+
+    :param dynamodb_client: Boto3 DynamoDB client
+    :param table_name: Name of the DynamoDB table to check
+    :return: True if the table exists, False otherwise
+    """
+    try:
+        dynamodb_client.describe_table(TableName=table_name)
+        return True
+    except ClientError as e:
+        if e.response['Error']['Code'] == 'ResourceNotFoundException':
+            print(f"Table '{table_name}' does not exist.")
+            return False
+        print(f"An error occurred while checking table '{table_name}': {e}.")
+        return False
+
+
+def validate_tables(dynamodb_client, operation, coordinator_state_table_name, lease_table_name=None):
+    """
+    Validate the existence of DynamoDB tables required for KCL operations
+
+    :param dynamodb_client: A boto3 DynamoDB client object
+    :param operation: Rollback or Roll-forward for logging
+    :param coordinator_state_table_name: Name of the coordinator state table
+    :param lease_table_name: Name of the DynamoDB KCL lease table (optional)
+    :return: True if all required tables exist, False otherwise
+    """
+    if lease_table_name and not table_exists(dynamodb_client, lease_table_name):
+        print(
+            f"{operation} failed. Could not find a KCL Application DDB lease table "
+            f"with name {lease_table_name}. Please pass in the correct application_name "
+            "and/or lease_table_name that matches your KCL application configuration."
+        )
+        return False
+
+    if not table_exists(dynamodb_client, coordinator_state_table_name):
+        print(
+            f"{operation} failed. Could not find a coordinator state table "
+            f"{coordinator_state_table_name}. Please pass in the correct application_name or"
+            f" coordinator_state_table_name that matches your KCL application configuration."
+        )
+        return False
+
+    return True
+
+
+def add_current_state_to_history(item, max_history=10):
+    """
+    Adds the current state of a DynamoDB item to its history attribute.
+    Creates a new history entry from the current value and maintains a capped history list.
+
+    :param item: DynamoDB item to add history to
+    :param max_history: Maximum number of history entries to maintain (default: 10)
+    :return: Updated history attribute as a DynamoDB-formatted dictionary
+    """
+    # Extract current values
+    current_version = item.get(CLIENT_VERSION_ATTR, {}).get('S', 'Unknown')
+    current_modified_by = item.get(MODIFIED_BY_ATTR, {}).get('S', 'Unknown')
+    current_time_in_millis = (
+        item.get(TIMESTAMP_ATTR, {}).get('N', get_time_in_millis())
+    )
+
+    # Create new history entry
+    new_entry = {
+        'M': {
+            CLIENT_VERSION_ATTR: {'S': current_version},
+            MODIFIED_BY_ATTR: {'S': current_modified_by},
+            TIMESTAMP_ATTR: {'N': current_time_in_millis}
+        }
+    }
+
+    # Get existing history or create new if doesn't exist
+    history_dict = item.get(f'{HISTORY_ATTR}', {'L': []})
+    history_list = history_dict['L']
+
+    # Add new entry to the beginning of the list, capping at max_history
+    history_list.insert(0, new_entry)
+    history_list = history_list[:max_history]
+
+    return history_dict
+
+
+def get_current_state(dynamodb_client, table_name):
+    """
+    Retrieve the current state from the DynamoDB table and prepare history update.
+    Fetches the current item from the specified DynamoDB table,
+    extracts the initial client version, and creates a new history entry.
+
+    :param dynamodb_client: Boto3 DynamoDB client
+    :param table_name: Name of the DynamoDB table to query
+    :return: A tuple containing:
+             - initial_version (str): The current client version, or 'Unknown' if not found
+             - new_history (dict): Updated history including the current state
+    """
+    response = dynamodb_client.get_item(
+        TableName=table_name,
+        Key={'key': {'S': MIGRATION_KEY}}
+    )
+    item = response.get('Item', {})
+    initial_version = item.get(CLIENT_VERSION_ATTR, {}).get('S', 'Unknown')
+    new_history = add_current_state_to_history(item)
+    return initial_version, new_history
+
+
+def rollback_client_version(dynamodb_client, table_name, history):
+    """
+    Update the client version in the coordinator state table to initiate rollback.
+
+    :param dynamodb_client: Boto3 DynamoDB client
+    :param table_name: Name of the coordinator state DDB table
+    :param history: Updated history attribute as a DynamoDB-formatted dictionary
+    :return: A tuple containing:
+             - success (bool): True if client version was successfully updated, False otherwise
+             - previous_version (str): The version that was replaced, or None if update failed
+    """
+    try:
+        print(f"Rolling back client version in table '{table_name}'...")
+        update_response = dynamodb_client.update_item(
+            TableName=table_name,
+            Key={'key': {'S': MIGRATION_KEY}},
+            UpdateExpression=(
+                f"SET {CLIENT_VERSION_ATTR} = :rollback_client_version, "
+                f"{TIMESTAMP_ATTR} = :updated_at, "
+                f"{MODIFIED_BY_ATTR} = :modifier, "
+                f"{HISTORY_ATTR} = :history"
+            ),
+            ConditionExpression=(
+                f"{CLIENT_VERSION_ATTR} IN ("
+                ":upgrade_from_2x_client_version, "
+                ":3x_with_rollback_client_version)"
+            ),
+            ExpressionAttributeValues={
+                ':rollback_client_version': {'S': KclClientVersion.VERSION_2X.value},
+                ':updated_at': {'N': get_time_in_millis()},
+                ':modifier': {'S': 'KclMigrationTool-rollback'},
+                ':history': history,
+                ':upgrade_from_2x_client_version': (
+                    {'S': KclClientVersion.UPGRADE_FROM_2X.value}
+                ),
+                ':3x_with_rollback_client_version': (
+                    {'S': KclClientVersion.VERSION_3X_WITH_ROLLBACK.value}
+                ),
+            },
+            ReturnValues='UPDATED_OLD'
+        )
+        replaced_item = update_response.get('Attributes', {})
+        replaced_version = replaced_item.get('cv', {}).get('S', '')
+        return True, replaced_version
+    except ClientError as e:
+        if e.response['Error']['Code'] == 'ConditionalCheckFailedException':
+            print("Unable to rollback, as application is not in a state that allows rollback."
+                  "Ensure that the given application_name or coordinator_state_table_name is correct and"
+                  " you have followed all prior migration steps.")
+        else:
+            print(f"An unexpected error occurred while rolling back: {str(e)}"
+                  "Please resolve and run this migration script again.")
+        return False, None
+
+
+def rollfoward_client_version(dynamodb_client, table_name, history):
+    """
+    Update the client version in the coordinator state table to initiate roll-forward
+    conditionally if application is currently in rolled back state.
+
+    :param dynamodb_client: Boto3 DynamoDB client
+    :param table_name: Name of the coordinator state DDB table
+    :param history: Updated history attribute as a DynamoDB-formatted dictionary
+    :return: True if client version was successfully updated, False otherwise
+    """
+    try:
+        # Conditionally update client version
+        dynamodb_client.update_item(
+            TableName=table_name,
+            Key={'key': {'S': MIGRATION_KEY}},
+            UpdateExpression= (
+                f"SET {CLIENT_VERSION_ATTR} = :rollforward_version, "
+                f"{TIMESTAMP_ATTR} = :updated_at, "
+                f"{MODIFIED_BY_ATTR} = :modifier, "
+                f"{HISTORY_ATTR} = :new_history"
+            ),
+            ConditionExpression=f"{CLIENT_VERSION_ATTR} = :kcl_2x_version",
+            ExpressionAttributeValues={
+                ':rollforward_version': {'S': KclClientVersion.UPGRADE_FROM_2X.value},
+                ':updated_at': {'N': get_time_in_millis()},
+                ':modifier': {'S': 'KclMigrationTool-rollforward'},
+                ':new_history': history,
+                ':kcl_2x_version': {'S': KclClientVersion.VERSION_2X.value},
+            }
+        )
+        print("Roll-forward has been initiated. KCL application will monitor for 3.0 readiness and"
+              " automatically switch to 3.0 functionality when readiness criteria have been met.")
+    except ClientError as e:
+        if e.response['Error']['Code'] == 'ConditionalCheckFailedException':
+            print("Unable to roll-forward because application is not in rolled back state."
+                  " Ensure that the given application_name or coordinator_state_table_name is correct"
+                  " and you have followed all prior migration steps.")
+        else:
+            print(f"Unable to roll-forward due to error: {str(e)}. "
+                  "Please resolve and run this migration script again.")
+    except Exception as e:
+        print(f"Unable to roll-forward due to error: {str(e)}. "
+              "Please resolve and run this migration script again.")
+
+
+def delete_gsi_if_exists(dynamodb_client, table_name):
+    """
+    Deletes GSI on given lease table if it exists.
+
+    :param dynamodb_client: Boto3 DynamoDB client
+    :param table_name: Name of lease table to remove GSI from
+    """
+    try:
+        gsi_present = False
+        response = dynamodb_client.describe_table(TableName=table_name)
+        if 'GlobalSecondaryIndexes' in response['Table']:
+            gsi_list = response['Table']['GlobalSecondaryIndexes']
+            for gsi in gsi_list:
+                if gsi['IndexName'] == GSI_NAME:
+                    gsi_present = True
+                    break
+
+        if not gsi_present:
+            print(f"GSI {GSI_NAME} is not present on lease table {table_name}. It may already be successfully"
+                  " deleted. Or if lease table name is incorrect, please re-run the KclMigrationTool with correct"
+                  " application_name or lease_table_name.")
+            return
+    except ClientError as e:
+        if e.response['Error']['Code'] == 'ResourceNotFoundException':
+            print(f"Lease table {table_name} does not exist, please check application_name or lease_table_name"
+                  " configuration and try again.")
+            return
+        else:
+            print(f"An unexpected error occurred while checking if GSI {GSI_NAME} exists"
+                  f" on lease table {table_name}: {str(e)}. Please rectify the error and try again.")
+            return
+
+    print(f"Deleting GSI '{GSI_NAME}' from table '{table_name}'...")
+    try:
+        dynamodb_client.update_table(
+            TableName=table_name,
+            GlobalSecondaryIndexUpdates=[
+                {
+                    'Delete': {
+                        'IndexName': GSI_NAME
+                    }
+                }
+            ]
+        )
+    except ClientError as e:
+        if e.response['Error']['Code'] == 'ResourceNotFoundException':
+            print(f"{GSI_NAME} not found or table '{table_name}' not found.")
+        elif e.response['Error']['Code'] == 'ResourceInUseException':
+            print(f"Unable to delete GSI: '{table_name}' is currently being modified.")
+    except Exception as e:
+        print(f"An unexpected error occurred while deleting GSI {GSI_NAME} on lease table {table_name}: {str(e)}."
+              " Please manually confirm the GSI is removed from the lease table, or"
+              " resolve the error and rerun the migration script.")
+
+
+def delete_worker_metrics_table_if_exists(dynamodb_client, worker_metrics_table_name):
+    """
+    Deletes worker metrics table based on application name, if it exists.
+
+    :param dynamodb_client: Boto3 DynamoDB client
+    :param worker_metrics_table_name: Name of the DynamoDB worker metrics table
+    """
+    try:
+        dynamodb_client.describe_table(TableName=worker_metrics_table_name)
+    except ClientError as e:
+        if e.response['Error']['Code'] == 'ResourceNotFoundException':
+            print(f"Worker metrics table {worker_metrics_table_name} does not exist."
+                  " It may already be successfully deleted. Please check that the application_name"
+                  " or worker_metrics_table_name is correct. If not, correct this and rerun the migration script.")
+            return
+        else:
+            print(f"An unexpected error occurred when checking if {worker_metrics_table_name} table exists: {str(e)}."
+                  " Please manually confirm the table is deleted, or resolve the error"
+                  " and rerun the migration script.")
+            return
+
+    print(f"Deleting worker metrics table {worker_metrics_table_name}...")
+    try:
+        dynamodb_client.delete_table(TableName=worker_metrics_table_name)
+    except ClientError as e:
+        if e.response['Error']['Code'] == 'AccessDeniedException':
+            print(f"No permissions to delete table {worker_metrics_table_name}. Please manually delete it if you"
+                  " want to avoid any charges until you are ready to rollforward with migration.")
+        else:
+            print(f"An unexpected client error occurred while deleting worker metrics table: {str(e)}."
+                  " Please manually confirm the table is deleted, or resolve the error"
+                  " and rerun the migration script.")
+    except Exception as e:
+        print(f"An unexpected error occurred while deleting worker metrics table: {str(e)}."
+              " Please manually confirm the table is deleted, or resolve the error"
+              " and rerun the migration script.")
+
+
+def perform_rollback(dynamodb_client, lease_table_name, coordinator_state_table_name, worker_metrics_table_name):
+    """
+    Perform KCL 3.0 migration rollback by updating MigrationState for the KCL application.
+    Rolls client version back, removes GSI from lease table, deletes worker metrics table.
+
+    :param dynamodb_client: Boto3 DynamoDB client
+    :param coordinator_state_table_name: Name of the DynamoDB coordinator state table
+    :param coordinator_state_table_name: Name of the DynamoDB coordinator state table
+    :param worker_metrics_table_name: Name of the DynamoDB worker metrics table
+    """
+    if not validate_tables(dynamodb_client, "Rollback", coordinator_state_table_name, lease_table_name):
+        return
+
+    try:
+        initial_version, new_history = get_current_state(dynamodb_client,
+                                                         coordinator_state_table_name)
+    except ClientError as e:
+        handle_get_item_client_error(e, "Rollback", coordinator_state_table_name)
+        return
+
+    if not is_valid_version(version=initial_version, mode='rollback'):
+        return
+
+    # 1. Rollback client version
+    if initial_version != KclClientVersion.VERSION_2X.value:
+        rollback_succeeded, initial_version = rollback_client_version(
+            dynamodb_client, coordinator_state_table_name, new_history
+        )
+        if not rollback_succeeded:
+            return
+
+    print(f"Waiting for {GSI_DELETION_WAIT_TIME_SECONDS} seconds before cleaning up KCL 3.0 resources after rollback...")
+    time.sleep(GSI_DELETION_WAIT_TIME_SECONDS)
+
+    # 2. Delete the GSI
+    delete_gsi_if_exists(dynamodb_client, lease_table_name)
+
+    # 3. Delete worker metrics table
+    delete_worker_metrics_table_if_exists(dynamodb_client, worker_metrics_table_name)
+
+    # Log success
+    if initial_version == KclClientVersion.UPGRADE_FROM_2X.value:
+        print("\nRollback completed. Your application was running 2x compatible functionality.")
+        print("Please rollback to your previous application binaries by deploying the code with your previous KCL version.")
+    elif initial_version == KclClientVersion.VERSION_3X_WITH_ROLLBACK.value:
+        print("\nRollback completed. Your KCL Application was running 3x functionality and will rollback to 2x compatible functionality.")
+        print("If you don't see mitigation after a short period of time,"
+              " please rollback to your previous application binaries by deploying the code with your previous KCL version.")
+    elif initial_version == KclClientVersion.VERSION_2X.value:
+        print("\nApplication was already rolled back. Any KCLv3 resources that could be deleted were cleaned up"
+              " to avoid charges until the application can be rolled forward with migration.")
+
+
+def perform_rollforward(dynamodb_client, coordinator_state_table_name):
+    """
+    Perform KCL 3.0 migration roll-forward by updating MigrationState for the KCL application
+
+    :param dynamodb_client: Boto3 DynamoDB client
+    :param coordinator_state_table_name: Name of the DynamoDB table
+    """
+    if not validate_tables(dynamodb_client, "Roll-forward", coordinator_state_table_name):
+        return
+
+    try:
+        initial_version, new_history = get_current_state(dynamodb_client,
+                                                         coordinator_state_table_name)
+    except ClientError as e:
+        handle_get_item_client_error(e, "Roll-forward", coordinator_state_table_name)
+        return
+
+    if not is_valid_version(version=initial_version, mode='rollforward'):
+        return
+
+    rollfoward_client_version(dynamodb_client, coordinator_state_table_name, new_history)
+
+
+def run_kcl_migration(mode, lease_table_name, coordinator_state_table_name, worker_metrics_table_name):
+    """
+    Update the MigrationState in CoordinatorState DDB Table
+
+    :param mode: Either 'rollback' or 'rollforward'
+    :param lease_table_name: Name of the DynamoDB KCL lease table
+    :param coordinator_state_table_name: Name of the DynamoDB coordinator state table
+    :param worker_metrics_table_name: Name of the DynamoDB worker metrics table
+    """
+    dynamodb_client = boto3.client('dynamodb', config=config)
+
+    if mode == "rollback":
+        perform_rollback(
+            dynamodb_client,
+            lease_table_name,
+            coordinator_state_table_name,
+            worker_metrics_table_name
+        )
+    elif mode == "rollforward":
+        perform_rollforward(dynamodb_client, coordinator_state_table_name)
+    else:
+        print(f"Invalid mode: {mode}. Please use 'rollback' or 'rollforward'.")
+
+
+def validate_args(args):
+    if args.mode == 'rollforward':
+        if not (args.application_name or args.coordinator_state_table_name):
+            raise ValueError(
+                "For rollforward mode, either application_name or "
+                "coordinator_state_table_name must be provided."
+            )
+    else:
+        if args.application_name:
+            return
+
+        if not (args.lease_table_name and
+                args.coordinator_state_table_name and
+                args.worker_metrics_table_name):
+            raise ValueError(
+                "For rollback mode, either application_name or all three table names "
+                "(lease_table_name, coordinator_state_table_name, and "
+                "worker_metrics_table_name) must be provided."
+            )
+
+def process_table_names(args):
+    """
+    Process command line arguments to determine table names based on mode.
+    Args:
+        args: Parsed command line arguments
+    Returns:
+        tuple: (mode, lease_table_name, coordinator_state_table_name, worker_metrics_table_name)
+    """
+    mode_input = args.mode
+    application_name_input = args.application_name
+
+    coordinator_state_table_name_input = (args.coordinator_state_table_name or
+                                          application_name_input + DEFAULT_COORDINATOR_STATE_TABLE_SUFFIX)
+    lease_table_name_input = None
+    worker_metrics_table_name_input = None
+
+    if mode_input == "rollback":
+        lease_table_name_input = args.lease_table_name or application_name_input
+        worker_metrics_table_name_input = (args.worker_metrics_table_name or
+                                           application_name_input + DEFAULT_WORKER_METRICS_TABLE_SUFFIX)
+
+    return (mode_input,
+            lease_table_name_input,
+            coordinator_state_table_name_input,
+            worker_metrics_table_name_input)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description=
+        """
+        KCL Migration Tool
+        This tool facilitates the migration and rollback processes for Amazon KCLv3 applications.
+    
+        Before running this tool:
+        1. Ensure you have the necessary AWS permissions configured to access and modify the following:
+            - KCL application DynamoDB tables (lease table and coordinator state table)
+    
+        2. Verify that your AWS credentials are properly set up in your environment or AWS config file.
+    
+        3. Confirm that you have the correct KCL application name and lease table name (if configured in KCL).
+    
+        Usage:
+        This tool supports two main operations: rollforward (upgrade) and rollback.
+        For detailed usage instructions, use the -h or --help option.
+        """,
+        formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--mode", choices=['rollback', 'rollforward'], required=True,
+                        help="Mode of operation: rollback or rollforward")
+    parser.add_argument("--application_name",
+                        help="Name of the KCL application. This must match the application name "
+                             "used in the KCL Library configurations.")
+    parser.add_argument("--lease_table_name",
+                        help="Name of the DynamoDB lease table (defaults to applicationName)."
+                             " If LeaseTable name was specified for the application as part of "
+                             "the KCL configurations, the same name must be passed here.")
+    parser.add_argument("--coordinator_state_table_name",
+                        help="Name of the DynamoDB coordinator state table "
+                             "(defaults to applicationName-CoordinatorState)."
+                             " If coordinator state table name was specified for the application "
+                             "as part of the KCL configurations, the same name must be passed here.")
+    parser.add_argument("--worker_metrics_table_name",
+                        help="Name of the DynamoDB worker metrics table "
+                             "(defaults to applicationName-WorkerMetricStats)."
+                             " If worker metrics table name was specified for the application "
+                             "as part of the KCL configurations, the same name must be passed here.")
+    parser.add_argument("--region", required=True,
+                        help="AWS Region where your KCL application exists")
+    args = parser.parse_args()
+    validate_args(args)
+    config.region_name = args.region
+    run_kcl_migration(*process_table_names(args))

amazon-kinesis-client/src/main/java-templates/KinesisClientLibraryPackage.java

@@ -0,0 +1,5 @@
+package software.amazon.kinesis.common;
+
+public final class KinesisClientLibraryPackage {
+    public static final String VERSION = "${project.version}";
+}

amazon-kinesis-client/src/main/java/software/amazon/kinesis/annotations/KinesisClientInternalApi.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.annotations;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+
+/**
+ * Marker interface for 'internal' APIs that should not be used outside the core module.
+ * Breaking changes can and will be introduced to elements marked as KinesisClientInternalApi.
+ * Users of the KCL should not depend on any packages, types, fields, constructors, or methods with this annotation.
+ */
+@Retention(RetentionPolicy.CLASS)
+public @interface KinesisClientInternalApi {}

amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/Checkpoint.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package software.amazon.kinesis.checkpoint;
+
+import lombok.Data;
+import lombok.experimental.Accessors;
+import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber;
+
+/**
+ * A class encapsulating the 2 pieces of state stored in a checkpoint.
+ */
+@Data
+@Accessors(fluent = true)
+public class Checkpoint {
+    private final ExtendedSequenceNumber checkpoint;
+    private final ExtendedSequenceNumber pendingCheckpoint;
+    private final byte[] pendingCheckpointState;
+
+    @Deprecated
+    public Checkpoint(final ExtendedSequenceNumber checkpoint, final ExtendedSequenceNumber pendingCheckpoint) {
+        this(checkpoint, pendingCheckpoint, null);
+    }
+
+    /**
+     * Constructor.
+     *
+     * @param checkpoint the checkpoint sequence number - cannot be null or empty.
+     * @param pendingCheckpoint the pending checkpoint sequence number - can be null.
+     * @param pendingCheckpointState the pending checkpoint state - can be null.
+     */
+    public Checkpoint(
+            final ExtendedSequenceNumber checkpoint,
+            final ExtendedSequenceNumber pendingCheckpoint,
+            byte[] pendingCheckpointState) {
+        if (checkpoint == null || checkpoint.sequenceNumber().isEmpty()) {
+            throw new IllegalArgumentException("Checkpoint cannot be null or empty");
+        }
+        this.checkpoint = checkpoint;
+        this.pendingCheckpoint = pendingCheckpoint;
+        this.pendingCheckpointState = pendingCheckpointState;
+    }
+}

amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/CheckpointConfig.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.checkpoint;
+
+import lombok.Data;
+import lombok.experimental.Accessors;
+import software.amazon.kinesis.checkpoint.dynamodb.DynamoDBCheckpointFactory;
+
+/**
+ * Used by the KCL to manage checkpointing.
+ */
+@Data
+@Accessors(fluent = true)
+public class CheckpointConfig {
+    private CheckpointFactory checkpointFactory = new DynamoDBCheckpointFactory();
+}

amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/CheckpointFactory.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.checkpoint;
+
+import software.amazon.kinesis.leases.LeaseCoordinator;
+import software.amazon.kinesis.leases.LeaseRefresher;
+import software.amazon.kinesis.processor.Checkpointer;
+
+/**
+ *
+ */
+public interface CheckpointFactory {
+    Checkpointer createCheckpointer(LeaseCoordinator leaseCoordinator, LeaseRefresher leaseRefresher);
+}

amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/DoesNothingPreparedCheckpointer.java

@@ -0,0 +1,66 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package software.amazon.kinesis.checkpoint;
+
+import software.amazon.kinesis.annotations.KinesisClientInternalApi;
+import software.amazon.kinesis.exceptions.InvalidStateException;
+import software.amazon.kinesis.exceptions.KinesisClientLibDependencyException;
+import software.amazon.kinesis.exceptions.ShutdownException;
+import software.amazon.kinesis.exceptions.ThrottlingException;
+import software.amazon.kinesis.processor.PreparedCheckpointer;
+import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber;
+
+/**
+ * A special PreparedCheckpointer that does nothing, which can be used when preparing a checkpoint at the current
+ * checkpoint sequence number where it is never necessary to do another checkpoint.
+ * This simplifies programming by preventing application developers from having to reason about whether
+ * their application has processed records before calling prepareCheckpoint
+ *
+ * Here's why it's safe to do nothing:
+ * The only way to checkpoint at current checkpoint value is to have a record processor that gets
+ * initialized, processes 0 records, then calls prepareCheckpoint(). The value in the table is the same, so there's
+ * no reason to overwrite it with another copy of itself.
+ */
+@KinesisClientInternalApi
+public class DoesNothingPreparedCheckpointer implements PreparedCheckpointer {
+
+    private final ExtendedSequenceNumber sequenceNumber;
+
+    /**
+     * Constructor.
+     * @param sequenceNumber the sequence number value
+     */
+    public DoesNothingPreparedCheckpointer(ExtendedSequenceNumber sequenceNumber) {
+        this.sequenceNumber = sequenceNumber;
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public ExtendedSequenceNumber pendingCheckpoint() {
+        return sequenceNumber;
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public void checkpoint()
+            throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException,
+                    IllegalArgumentException {
+        // This method does nothing
+    }
+}

amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/SentinelCheckpoint.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package software.amazon.kinesis.checkpoint;
+
+/**
+ * Enumeration of the sentinel values of checkpoints.
+ * Used during initialization of ShardConsumers to determine the starting point
+ * in the shard and to flag that a shard has been completely processed.
+ */
+public enum SentinelCheckpoint {
+    /**
+     * Start from the first available record in the shard.
+     */
+    TRIM_HORIZON,
+    /**
+     * Start from the latest record in the shard.
+     */
+    LATEST,
+    /**
+     * We've completely processed all records in this shard.
+     */
+    SHARD_END,
+    /**
+     * Start from the record at or after the specified server-side timestamp.
+     */
+    AT_TIMESTAMP
+}

amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/SequenceNumberValidator.java

@@ -0,0 +1,190 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.checkpoint;
+
+import java.math.BigInteger;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+
+import lombok.Data;
+import lombok.experimental.Accessors;
+import org.apache.commons.lang3.StringUtils;
+
+/**
+ * This supports extracting the shardId from a sequence number.
+ *
+ * <h2>Warning</h2>
+ * <strong>Sequence numbers are an opaque value used by Kinesis, and maybe changed at any time. Should validation stop
+ * working you may need to update your version of the KCL</strong>
+ *
+ */
+public class SequenceNumberValidator {
+
+    @Data
+    @Accessors(fluent = true)
+    private static class SequenceNumberComponents {
+        final int version;
+        final int shardId;
+    }
+
+    private interface SequenceNumberReader {
+        Optional<SequenceNumberComponents> read(String sequenceNumber);
+    }
+
+    /**
+     * Reader for the v2 sequence number format. v1 sequence numbers are no longer used or available.
+     */
+    private static class V2SequenceNumberReader implements SequenceNumberReader {
+
+        private static final int VERSION = 2;
+
+        private static final int EXPECTED_BIT_LENGTH = 186;
+
+        private static final int VERSION_OFFSET = 184;
+        private static final long VERSION_MASK = (1 << 4) - 1;
+
+        private static final int SHARD_ID_OFFSET = 4;
+        private static final long SHARD_ID_MASK = (1L << 32) - 1;
+
+        @Override
+        public Optional<SequenceNumberComponents> read(String sequenceNumberString) {
+            BigInteger sequenceNumber = new BigInteger(sequenceNumberString, 10);
+
+            //
+            // If the bit length of the sequence number isn't 186 it's impossible for the version numbers
+            // to be where we expect them. We treat this the same as an unknown version of the sequence number
+            //
+            // If the sequence number length isn't what we expect it's due to a new version of the sequence number or
+            // an invalid sequence number. This
+            //
+            if (sequenceNumber.bitLength() != EXPECTED_BIT_LENGTH) {
+                return Optional.empty();
+            }
+
+            //
+            // Read the 4 most significant bits of the sequence number, the 2 most significant bits are implicitly 0
+            // (2 == 0b0011). If the version number doesn't match we give up and say we can't parse the sequence number
+            //
+            int version = readOffset(sequenceNumber, VERSION_OFFSET, VERSION_MASK);
+            if (version != VERSION) {
+                return Optional.empty();
+            }
+
+            //
+            // If we get here the sequence number is big enough, and the version matches so the shardId should be valid.
+            //
+            int shardId = readOffset(sequenceNumber, SHARD_ID_OFFSET, SHARD_ID_MASK);
+            return Optional.of(new SequenceNumberComponents(version, shardId));
+        }
+
+        private int readOffset(BigInteger sequenceNumber, int offset, long mask) {
+            long value = sequenceNumber.shiftRight(offset).longValue() & mask;
+            return (int) value;
+        }
+    }
+
+    private static final List<SequenceNumberReader> SEQUENCE_NUMBER_READERS =
+            Collections.singletonList(new V2SequenceNumberReader());
+
+    private Optional<SequenceNumberComponents> retrieveComponentsFor(String sequenceNumber) {
+        return SEQUENCE_NUMBER_READERS.stream()
+                .map(r -> r.read(sequenceNumber))
+                .filter(Optional::isPresent)
+                .map(Optional::get)
+                .findFirst();
+    }
+
+    /**
+     * Attempts to retrieve the version for a sequence number. If no reader can be found for the sequence number this
+     * will return an empty Optional.
+     *
+     * <p>
+     * <strong>This will return an empty Optional if the it's unable to extract the version number. This can occur for
+     * multiple reasons including:
+     * <ul>
+     * <li>Kinesis has started using a new version of sequence numbers</li>
+     * <li>The provided sequence number isn't a valid Kinesis sequence number.</li>
+     * </ul>
+     * </strong>
+     * </p>
+     *
+     * @param sequenceNumber
+     *            the sequence number to extract the version from
+     * @return an Optional containing the version if a compatible sequence number reader can be found, an empty Optional
+     *         otherwise.
+     */
+    public Optional<Integer> versionFor(String sequenceNumber) {
+        return retrieveComponentsFor(sequenceNumber).map(SequenceNumberComponents::version);
+    }
+
+    /**
+     * Attempts to retrieve the shardId from a sequence number. If the version of the sequence number is unsupported
+     * this will return an empty optional.
+     *
+     * <strong>This will return an empty Optional if the sequence number isn't recognized. This can occur for multiple
+     * reasons including:
+     * <ul>
+     * <li>Kinesis has started using a new version of sequence numbers</li>
+     * <li>The provided sequence number isn't a valid Kinesis sequence number.</li>
+     * </ul>
+     * </strong>
+     * <p>
+     * This should always return a value if {@link #versionFor(String)} returns a value
+     * </p>
+     *
+     * @param sequenceNumber
+     *            the sequence number to extract the shardId from
+     * @return an Optional containing the shardId if the version is supported, an empty Optional otherwise.
+     */
+    public Optional<String> shardIdFor(String sequenceNumber) {
+        return retrieveComponentsFor(sequenceNumber).map(s -> String.format("shardId-%012d", s.shardId()));
+    }
+
+    /**
+     * Validates that the sequence number provided contains the given shardId. If the sequence number is unsupported
+     * this will return an empty Optional.
+     *
+     * <p>
+     * Validation of a sequence number will only occur if the sequence number can be parsed. It's possible to use
+     * {@link #versionFor(String)} to verify that the given sequence number is supported by this class. There are 3
+     * possible validation states:
+     * <dl>
+     * <dt>Some(True)</dt>
+     * <dd>The sequence number can be parsed, and the shardId matches the one in the sequence number</dd>
+     * <dt>Some(False)</dt>
+     * <dd>THe sequence number can be parsed, and the shardId doesn't match the one in the sequence number</dd>
+     * <dt>None</dt>
+     * <dd>It wasn't possible to parse the sequence number so the validity of the sequence number is unknown</dd>
+     * </dl>
+     * </p>
+     *
+     * <p>
+     * <strong>Handling unknown validation causes is application specific, and not specific handling is
+     * provided.</strong>
+     * </p>
+     *
+     * @param sequenceNumber
+     *            the sequence number to verify the shardId
+     * @param shardId
+     *            the shardId that the sequence is expected to contain
+     * @return true if the sequence number contains the shardId, false if it doesn't. If the sequence number version is
+     *         unsupported this will return an empty Optional
+     */
+    public Optional<Boolean> validateSequenceNumberForShard(String sequenceNumber, String shardId) {
+        return shardIdFor(sequenceNumber).map(s -> StringUtils.equalsIgnoreCase(s, shardId));
+    }
+}

amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/ShardPreparedCheckpointer.java

@@ -0,0 +1,65 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package software.amazon.kinesis.checkpoint;
+
+import software.amazon.kinesis.exceptions.InvalidStateException;
+import software.amazon.kinesis.exceptions.KinesisClientLibDependencyException;
+import software.amazon.kinesis.exceptions.ShutdownException;
+import software.amazon.kinesis.exceptions.ThrottlingException;
+import software.amazon.kinesis.processor.PreparedCheckpointer;
+import software.amazon.kinesis.processor.RecordProcessorCheckpointer;
+import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber;
+
+/**
+ * Objects of this class are prepared to checkpoint at a specific sequence number. They use an
+ * RecordProcessorCheckpointer to do the actual checkpointing, so their checkpoint is subject to the same 'didn't go
+ * backwards' validation as a normal checkpoint.
+ */
+public class ShardPreparedCheckpointer implements PreparedCheckpointer {
+
+    private final ExtendedSequenceNumber pendingCheckpointSequenceNumber;
+    private final RecordProcessorCheckpointer checkpointer;
+
+    /**
+     * Constructor.
+     *
+     * @param pendingCheckpointSequenceNumber sequence number to checkpoint at
+     * @param checkpointer checkpointer to use
+     */
+    public ShardPreparedCheckpointer(
+            ExtendedSequenceNumber pendingCheckpointSequenceNumber, RecordProcessorCheckpointer checkpointer) {
+        this.pendingCheckpointSequenceNumber = pendingCheckpointSequenceNumber;
+        this.checkpointer = checkpointer;
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public ExtendedSequenceNumber pendingCheckpoint() {
+        return pendingCheckpointSequenceNumber;
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public void checkpoint()
+            throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException,
+                    IllegalArgumentException {
+        checkpointer.checkpoint(
+                pendingCheckpointSequenceNumber.sequenceNumber(), pendingCheckpointSequenceNumber.subSequenceNumber());
+    }
+}

amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/ShardRecordProcessorCheckpointer.java

@@ -0,0 +1,388 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package software.amazon.kinesis.checkpoint;
+
+import lombok.Getter;
+import lombok.NonNull;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.Accessors;
+import lombok.extern.slf4j.Slf4j;
+import software.amazon.awssdk.services.kinesis.model.Record;
+import software.amazon.kinesis.exceptions.InvalidStateException;
+import software.amazon.kinesis.exceptions.KinesisClientLibDependencyException;
+import software.amazon.kinesis.exceptions.KinesisClientLibException;
+import software.amazon.kinesis.exceptions.ShutdownException;
+import software.amazon.kinesis.exceptions.ThrottlingException;
+import software.amazon.kinesis.leases.ShardInfo;
+import software.amazon.kinesis.processor.Checkpointer;
+import software.amazon.kinesis.processor.PreparedCheckpointer;
+import software.amazon.kinesis.processor.RecordProcessorCheckpointer;
+import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber;
+
+/**
+ * This class is used to enable RecordProcessors to checkpoint their progress.
+ * The Amazon Kinesis Client Library will instantiate an object and provide a reference to the application
+ * ShardRecordProcessor instance. Amazon Kinesis Client Library will create one instance per shard assignment.
+ */
+@RequiredArgsConstructor
+@Slf4j
+public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpointer {
+    @NonNull
+    private final ShardInfo shardInfo;
+
+    @NonNull
+    @Getter
+    @Accessors(fluent = true)
+    private final Checkpointer checkpointer;
+
+    // Set to the last value set via checkpoint().
+    // Sample use: verify application shutdown() invoked checkpoint() at the end of a shard.
+    @Getter
+    @Accessors(fluent = true)
+    private ExtendedSequenceNumber lastCheckpointValue;
+
+    @Getter
+    @Accessors(fluent = true)
+    private ExtendedSequenceNumber largestPermittedCheckpointValue;
+
+    private ExtendedSequenceNumber sequenceNumberAtShardEnd;
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public synchronized void checkpoint()
+            throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
+        if (log.isDebugEnabled()) {
+            log.debug(
+                    "Checkpointing {}, token {} at largest permitted value {}",
+                    ShardInfo.getLeaseKey(shardInfo),
+                    shardInfo.concurrencyToken(),
+                    this.largestPermittedCheckpointValue);
+        }
+        advancePosition(this.largestPermittedCheckpointValue);
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public synchronized void checkpoint(Record record)
+            throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException,
+                    IllegalArgumentException {
+
+        // TODO: UserRecord Deprecation
+        if (record == null) {
+            throw new IllegalArgumentException("Could not checkpoint a null record");
+        } /* else if (record instanceof UserRecord) {
+              checkpoint(record.sequenceNumber(), ((UserRecord) record).subSequenceNumber());
+          } */ else {
+            checkpoint(record.sequenceNumber(), 0);
+        }
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public synchronized void checkpoint(String sequenceNumber)
+            throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException,
+                    IllegalArgumentException {
+        checkpoint(sequenceNumber, 0);
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public synchronized void checkpoint(String sequenceNumber, long subSequenceNumber)
+            throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException,
+                    IllegalArgumentException {
+
+        if (subSequenceNumber < 0) {
+            throw new IllegalArgumentException(
+                    "Could not checkpoint at invalid, negative subsequence number " + subSequenceNumber);
+        }
+
+        /*
+         * If there isn't a last checkpoint value, we only care about checking the upper bound.
+         * If there is a last checkpoint value, we want to check both the lower and upper bound.
+         */
+        ExtendedSequenceNumber newCheckpoint = new ExtendedSequenceNumber(sequenceNumber, subSequenceNumber);
+        if ((lastCheckpointValue == null || lastCheckpointValue.compareTo(newCheckpoint) <= 0)
+                && newCheckpoint.compareTo(largestPermittedCheckpointValue) <= 0) {
+
+            if (log.isDebugEnabled()) {
+                log.debug(
+                        "Checkpointing {}, token {} at specific extended sequence number {}",
+                        ShardInfo.getLeaseKey(shardInfo),
+                        shardInfo.concurrencyToken(),
+                        newCheckpoint);
+            }
+            this.advancePosition(newCheckpoint);
+        } else {
+            throw new IllegalArgumentException(String.format(
+                    "Could not checkpoint at extended sequence number %s as it did not fall into acceptable range "
+                            + "between the last checkpoint %s and the greatest extended sequence number passed to this "
+                            + "record processor %s",
+                    newCheckpoint, this.lastCheckpointValue, this.largestPermittedCheckpointValue));
+        }
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public synchronized PreparedCheckpointer prepareCheckpoint()
+            throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
+        return this.prepareCheckpoint(
+                this.largestPermittedCheckpointValue.sequenceNumber(),
+                this.largestPermittedCheckpointValue.subSequenceNumber());
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public PreparedCheckpointer prepareCheckpoint(byte[] applicationState)
+            throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
+        return prepareCheckpoint(largestPermittedCheckpointValue.sequenceNumber(), applicationState);
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public PreparedCheckpointer prepareCheckpoint(Record record, byte[] applicationState)
+            throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
+        //
+        // TODO: UserRecord Deprecation
+        //
+        if (record == null) {
+            throw new IllegalArgumentException("Could not prepare checkpoint a null record");
+        } /*else if (record instanceof UserRecord) {
+              return prepareCheckpoint(record.sequenceNumber(), ((UserRecord) record).subSequenceNumber());
+          } */ else {
+            return prepareCheckpoint(record.sequenceNumber(), 0, applicationState);
+        }
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public synchronized PreparedCheckpointer prepareCheckpoint(Record record)
+            throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
+        return prepareCheckpoint(record, null);
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public synchronized PreparedCheckpointer prepareCheckpoint(String sequenceNumber)
+            throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
+        return prepareCheckpoint(sequenceNumber, 0);
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public PreparedCheckpointer prepareCheckpoint(String sequenceNumber, byte[] applicationState)
+            throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException,
+                    IllegalArgumentException {
+        return prepareCheckpoint(sequenceNumber, 0, applicationState);
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public synchronized PreparedCheckpointer prepareCheckpoint(String sequenceNumber, long subSequenceNumber)
+            throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
+        return prepareCheckpoint(sequenceNumber, subSequenceNumber, null);
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public PreparedCheckpointer prepareCheckpoint(
+            String sequenceNumber, long subSequenceNumber, byte[] applicationState)
+            throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException,
+                    IllegalArgumentException {
+        if (subSequenceNumber < 0) {
+            throw new IllegalArgumentException(
+                    "Could not checkpoint at invalid, negative subsequence number " + subSequenceNumber);
+        }
+
+        /*
+         * If there isn't a last checkpoint value, we only care about checking the upper bound.
+         * If there is a last checkpoint value, we want to check both the lower and upper bound.
+         */
+        ExtendedSequenceNumber pendingCheckpoint = new ExtendedSequenceNumber(sequenceNumber, subSequenceNumber);
+        if ((lastCheckpointValue == null || lastCheckpointValue.compareTo(pendingCheckpoint) <= 0)
+                && pendingCheckpoint.compareTo(largestPermittedCheckpointValue) <= 0) {
+
+            if (log.isDebugEnabled()) {
+                log.debug(
+                        "Preparing checkpoint {}, token {} at specific extended sequence number {}",
+                        ShardInfo.getLeaseKey(shardInfo),
+                        shardInfo.concurrencyToken(),
+                        pendingCheckpoint);
+            }
+            return doPrepareCheckpoint(pendingCheckpoint, applicationState);
+        } else {
+            throw new IllegalArgumentException(String.format(
+                    "Could not prepare checkpoint at extended sequence number %s as it did not fall into acceptable "
+                            + "range between the last checkpoint %s and the greatest extended sequence number passed "
+                            + "to this record processor %s",
+                    pendingCheckpoint, this.lastCheckpointValue, this.largestPermittedCheckpointValue));
+        }
+    }
+
+    public synchronized void setInitialCheckpointValue(ExtendedSequenceNumber initialCheckpoint) {
+        lastCheckpointValue = initialCheckpoint;
+    }
+
+    /**
+     * @param largestPermittedCheckpointValue the largest permitted checkpoint
+     */
+    public synchronized void largestPermittedCheckpointValue(ExtendedSequenceNumber largestPermittedCheckpointValue) {
+        this.largestPermittedCheckpointValue = largestPermittedCheckpointValue;
+    }
+
+    /**
+     * Used to remember the last extended sequence number before SHARD_END to allow us to prevent the checkpointer
+     * from checkpointing at the end of the shard twice (i.e. at the last extended sequence number and then again
+     * at SHARD_END).
+     *
+     * @param extendedSequenceNumber
+     */
+    public synchronized void sequenceNumberAtShardEnd(ExtendedSequenceNumber extendedSequenceNumber) {
+        this.sequenceNumberAtShardEnd = extendedSequenceNumber;
+    }
+
+    /**
+     * Internal API - has package level access only for testing purposes.
+     *
+     * @param sequenceNumber
+     *
+     * @throws KinesisClientLibDependencyException
+     * @throws ThrottlingException
+     * @throws ShutdownException
+     * @throws InvalidStateException
+     */
+    void advancePosition(String sequenceNumber)
+            throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
+        advancePosition(new ExtendedSequenceNumber(sequenceNumber));
+    }
+
+    void advancePosition(ExtendedSequenceNumber extendedSequenceNumber)
+            throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
+        ExtendedSequenceNumber checkpointToRecord = extendedSequenceNumber;
+        if (sequenceNumberAtShardEnd != null && sequenceNumberAtShardEnd.equals(extendedSequenceNumber)) {
+            // If we are about to checkpoint the very last sequence number for this shard, we might as well
+            // just checkpoint at SHARD_END
+            checkpointToRecord = ExtendedSequenceNumber.SHARD_END;
+        }
+
+        // Don't checkpoint a value we already successfully checkpointed
+        if (extendedSequenceNumber != null && !extendedSequenceNumber.equals(lastCheckpointValue)) {
+            try {
+                if (log.isDebugEnabled()) {
+                    log.debug(
+                            "Setting {}, token {} checkpoint to {}",
+                            ShardInfo.getLeaseKey(shardInfo),
+                            shardInfo.concurrencyToken(),
+                            checkpointToRecord);
+                }
+                checkpointer.setCheckpoint(
+                        ShardInfo.getLeaseKey(shardInfo), checkpointToRecord, shardInfo.concurrencyToken());
+                lastCheckpointValue = checkpointToRecord;
+            } catch (ThrottlingException
+                    | ShutdownException
+                    | InvalidStateException
+                    | KinesisClientLibDependencyException e) {
+                throw e;
+            } catch (KinesisClientLibException e) {
+                log.warn("Caught exception setting checkpoint.", e);
+                throw new KinesisClientLibDependencyException("Caught exception while checkpointing", e);
+            }
+        }
+    }
+
+    /**
+     * This method stores the given sequenceNumber as a pending checkpoint in the lease table without overwriting the
+     * current checkpoint, then returns a PreparedCheckpointer that is ready to checkpoint at the given sequence number.
+     *
+     * This method does not advance lastCheckpointValue, but calls to PreparedCheckpointer.checkpoint() on the returned
+     * objects do. This allows customers to 'discard' prepared checkpoints by calling any of the 4 checkpoint methods on
+     * this class before calling PreparedCheckpointer.checkpoint(). Some examples:
+     *
+     * 1) prepareCheckpoint(snA); checkpoint(snB). // this works regardless of whether snA or snB is bigger. It discards
+     * the prepared checkpoint at snA.
+     * 2) prepareCheckpoint(snA); prepareCheckpoint(snB). // this works regardless of whether snA or snB is bigger. It
+     * replaces the preparedCheckpoint at snA with a new one at snB.
+     * 3) checkpointA = prepareCheckpoint(snA); checkpointB = prepareCheckpoint(snB); checkpointB.checkpoint();
+     * checkpointerA.checkpoint(); // This replaces the prepared checkpoint at snA with a new one at snB, then
+     * checkpoints at snB regardless of whether snA or snB is bigger. The checkpoint at snA only succeeds if snA > snB.
+     *
+     * @param extendedSequenceNumber the sequence number for the prepared checkpoint
+     * @return a prepared checkpoint that is ready to checkpoint at the given sequence number.
+     * @throws KinesisClientLibDependencyException
+     * @throws InvalidStateException
+     * @throws ThrottlingException
+     * @throws ShutdownException
+     */
+    private PreparedCheckpointer doPrepareCheckpoint(
+            ExtendedSequenceNumber extendedSequenceNumber, byte[] applicationState)
+            throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
+
+        ExtendedSequenceNumber newPrepareCheckpoint = extendedSequenceNumber;
+        if (sequenceNumberAtShardEnd != null && sequenceNumberAtShardEnd.equals(extendedSequenceNumber)) {
+            // If we are about to checkpoint the very last sequence number for this shard, we might as well
+            // just checkpoint at SHARD_END
+            newPrepareCheckpoint = ExtendedSequenceNumber.SHARD_END;
+        }
+
+        // Don't actually prepare a checkpoint if they're trying to checkpoint at the current checkpointed value.
+        // The only way this can happen is if they call prepareCheckpoint() in a record processor that was initialized
+        // AND that has not processed any records since initialization.
+        if (newPrepareCheckpoint.equals(lastCheckpointValue)) {
+            return new DoesNothingPreparedCheckpointer(newPrepareCheckpoint);
+        }
+
+        try {
+            checkpointer.prepareCheckpoint(
+                    ShardInfo.getLeaseKey(shardInfo),
+                    newPrepareCheckpoint,
+                    shardInfo.concurrencyToken(),
+                    applicationState);
+        } catch (ThrottlingException
+                | ShutdownException
+                | InvalidStateException
+                | KinesisClientLibDependencyException e) {
+            throw e;
+        } catch (KinesisClientLibException e) {
+            log.warn("Caught exception setting prepareCheckpoint.", e);
+            throw new KinesisClientLibDependencyException("Caught exception while prepareCheckpointing", e);
+        }
+
+        ShardPreparedCheckpointer result = new ShardPreparedCheckpointer(newPrepareCheckpoint, this);
+        return result;
+    }
+}

amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/dynamodb/DynamoDBCheckpointFactory.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.checkpoint.dynamodb;
+
+import lombok.Data;
+import software.amazon.kinesis.annotations.KinesisClientInternalApi;
+import software.amazon.kinesis.checkpoint.CheckpointFactory;
+import software.amazon.kinesis.leases.LeaseCoordinator;
+import software.amazon.kinesis.leases.LeaseRefresher;
+import software.amazon.kinesis.processor.Checkpointer;
+
+/**
+ *
+ */
+@Data
+@KinesisClientInternalApi
+public class DynamoDBCheckpointFactory implements CheckpointFactory {
+    @Override
+    public Checkpointer createCheckpointer(
+            final LeaseCoordinator leaseLeaseCoordinator, final LeaseRefresher leaseRefresher) {
+        return new DynamoDBCheckpointer(leaseLeaseCoordinator, leaseRefresher);
+    }
+}