Compare commits
115 commits
kcl_releas
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
37ae2f86be | ||
|
|
1ce6123a78 | ||
|
|
ca0b0c15e1 | ||
|
|
63911b53a0 | ||
|
|
d7dd21beca | ||
|
|
897d993782 | ||
|
|
6990fc513f | ||
|
|
133374706c | ||
|
|
edd7d9b1e5 | ||
|
|
458ed653bc | ||
|
|
9daed2dda9 | ||
|
|
5263b4227c | ||
|
|
7b5ebaeb93 | ||
|
|
e856e7c95e | ||
|
|
c9563ab585 | ||
|
|
8deebe4bda | ||
|
|
68a7a9bf53 | ||
|
|
ae9a433ebd | ||
|
|
854d99da52 | ||
|
|
d9a238d33d | ||
|
|
dcac19e927 | ||
|
|
9da91cd43c | ||
|
|
4492e1e206 | ||
|
|
01d0e5cc55 | ||
|
|
3facf303bf | ||
|
|
b154acf7f5 | ||
|
|
004f6d3ea0 | ||
|
|
f7e909c979 | ||
|
|
fbfd74df39 | ||
|
|
a159fa31fb | ||
|
|
0478575b3d | ||
|
|
1c0c41c4e8 | ||
|
|
5878ba8ac6 | ||
|
|
3b5c59ec04 | ||
|
|
57dcddf10b | ||
|
|
f507066ea6 | ||
|
|
6cba7f431d | ||
|
|
715690d2c0 | ||
|
|
78fb42ede1 | ||
|
|
b271fed18b | ||
|
|
cbed8c3c6f | ||
|
|
e8c3c01b12 | ||
|
|
a83b3fbd57 | ||
|
|
0005670513 | ||
|
|
35b150009a | ||
|
|
d3b84dc995 | ||
|
|
49fa32d68e | ||
|
|
221db0564d | ||
|
|
adcc89f80d | ||
|
|
18fe49eed0 | ||
|
|
409a8f13cd | ||
|
|
c5e4fa2de0 | ||
|
|
a8abbea29f | ||
|
|
2648123a79 | ||
|
|
023f6be251 | ||
|
|
71f31e0a11 | ||
|
|
16e8404dc4 | ||
|
|
69cf5996c5 | ||
|
|
35fc72b2c8 | ||
|
|
89a90e34e2 | ||
|
|
ab24b66039 | ||
|
|
34fe58c492 | ||
|
|
ec34ed1def | ||
|
|
c12cee2a1b | ||
|
|
96be30b3e7 | ||
|
|
7d55b7a77f | ||
|
|
e9990190cc | ||
|
|
969341130a | ||
|
|
981899499f | ||
|
|
7f1f243676 | ||
|
|
581d713815 | ||
|
|
24774bc2e3 | ||
|
|
bf5ab60f4b | ||
|
|
44beda5cb7 | ||
|
|
38b8eb5fa2 | ||
|
|
c6b5d6872d | ||
|
|
fcfa3ec08f | ||
|
|
e9b810ba07 | ||
|
|
f205673fee | ||
|
|
a95aa9e39e | ||
|
|
ddff314020 | ||
|
|
f6df50ef6e | ||
|
|
f16217f1a9 | ||
|
|
63ba724df7 | ||
|
|
35b7e73514 | ||
|
|
1675f0a297 | ||
|
|
1280325c20 | ||
|
|
63e0fe7537 | ||
|
|
b4c2c6c947 | ||
|
|
b9ec494b02 | ||
|
|
5f3de14c88 | ||
|
|
b2eb38e510 | ||
|
|
2d769733fe | ||
|
|
fb6ab3f0bc | ||
|
|
a5d1c67660 | ||
|
|
1727765d87 | ||
|
|
1701eb3763 | ||
|
|
44837b702a | ||
|
|
a48f5436ee | ||
|
|
51a62a559c | ||
|
|
8ed4999a46 | ||
|
|
a7694a1f67 | ||
|
|
f90b1b1c05 | ||
|
|
118783b18b | ||
|
|
cf5e1e4c7f | ||
|
|
7899820cb1 | ||
|
|
0aff42f8fe | ||
|
|
aabcbaf4b7 | ||
|
|
78b565fa9b | ||
|
|
a1731dc49b | ||
|
|
7384bc1dbe | ||
|
|
12b9a36d0d | ||
|
|
2f4ff65681 | ||
|
|
46cd1179d4 | ||
|
|
eccd6cf2e7 |
485 changed files with 37599 additions and 13185 deletions
1
.git-blame-ignore-revs
Normal file
1
.git-blame-ignore-revs
Normal file
|
|
@ -0,0 +1 @@
|
|||
63ff312818a5f70eab9ec5bf80b53bdd7bf80248
|
||||
13
.github/dependabot.yml
vendored
13
.github/dependabot.yml
vendored
|
|
@ -1,10 +1,22 @@
|
|||
version: 2
|
||||
updates:
|
||||
# branch - master
|
||||
- package-ecosystem: "maven"
|
||||
directory: "/"
|
||||
labels:
|
||||
- "dependencies"
|
||||
- "v3.x"
|
||||
target-branch: "master"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
|
||||
# branch - v2.x
|
||||
- package-ecosystem: "maven"
|
||||
directory: "/"
|
||||
labels:
|
||||
- "dependencies"
|
||||
- "v2.x"
|
||||
target-branch: "v2.x"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
|
||||
|
|
@ -17,4 +29,3 @@ updates:
|
|||
target-branch: "v1.x"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
|
||||
|
|
|
|||
144
.github/scripts/backwards_compatibility_check.sh
vendored
Executable file
144
.github/scripts/backwards_compatibility_check.sh
vendored
Executable file
|
|
@ -0,0 +1,144 @@
|
|||
#!/bin/bash
|
||||
|
||||
TRUE=1
|
||||
FALSE=0
|
||||
KCL_MAVEN_DIR=~/.m2/repository/software/amazon/kinesis/amazon-kinesis-client
|
||||
|
||||
REMOVED_METHODS_FLAG=$FALSE
|
||||
LATEST_VERSION=""
|
||||
LATEST_JAR=""
|
||||
CURRENT_VERSION=""
|
||||
CURRENT_JAR=""
|
||||
|
||||
# Get the JAR from the latest version release on Maven.
|
||||
get_latest_jar() {
|
||||
# clear the directory so that the latest release will be the only version in the Maven directory after running mvn dependency:get
|
||||
rm -rf "$KCL_MAVEN_DIR"
|
||||
mvn -B dependency:get -Dartifact=software.amazon.kinesis:amazon-kinesis-client:LATEST
|
||||
LATEST_VERSION=$(ls "$KCL_MAVEN_DIR" | grep -E '[0-9]+.[0-9]+.[0-9]+')
|
||||
LATEST_JAR=$KCL_MAVEN_DIR/$LATEST_VERSION/amazon-kinesis-client-$LATEST_VERSION.jar
|
||||
}
|
||||
|
||||
# Get the JAR with the changes that need to be verified.
|
||||
get_current_jar() {
|
||||
mvn -B install -Dmaven.test.skip=true
|
||||
CURRENT_VERSION=$(mvn -q -Dexec.executable=echo -Dexec.args='${project.version}' --non-recursive exec:exec)
|
||||
CURRENT_JAR=$KCL_MAVEN_DIR/$CURRENT_VERSION/amazon-kinesis-client-$CURRENT_VERSION.jar
|
||||
}
|
||||
|
||||
is_new_minor_release() {
|
||||
is_new_major_release && return 1
|
||||
|
||||
local latest_minor_version=$(echo "$LATEST_VERSION" | cut -d . -f 2)
|
||||
local current_minor_version=$(echo "$CURRENT_VERSION" | cut -d . -f 2)
|
||||
[[ "$latest_minor_version" != "$current_minor_version" ]]
|
||||
return $?
|
||||
}
|
||||
|
||||
is_new_major_release() {
|
||||
local latest_major_version=$(echo "$LATEST_VERSION" | cut -d . -f 1)
|
||||
local current_major_version=$(echo "$CURRENT_VERSION" | cut -d . -f 1)
|
||||
[[ "$latest_major_version" != "$current_major_version" ]]
|
||||
return $?
|
||||
}
|
||||
|
||||
# Skip classes with the KinesisClientInternalApi annotation. These classes are subject to breaking backwards compatibility.
|
||||
is_kinesis_client_internal_api() {
|
||||
local current_class="$1"
|
||||
local grep_internal_api_result=$(javap -v -classpath "$LATEST_JAR" "$current_class" | grep KinesisClientInternalApi)
|
||||
[[ "$grep_internal_api_result" != "" ]]
|
||||
return $?
|
||||
}
|
||||
|
||||
# Skip classes which are not public (e.g. package level). These classes will not break backwards compatibility.
|
||||
is_non_public_class() {
|
||||
local current_class="$1"
|
||||
local class_definition=$(javap -classpath "$LATEST_JAR" "$current_class" | head -2 | tail -1)
|
||||
[[ "$class_definition" != *"public"* ]]
|
||||
return $?
|
||||
}
|
||||
|
||||
# Ignore methods that change from abstract to non-abstract (and vice-versa) if the class is an interface.\
|
||||
# Ignore methods that change from synchronized to non-synchronized (and vice-versa)
|
||||
ignore_non_breaking_changes() {
|
||||
local current_class="$1"
|
||||
local class_definition=$(javap -classpath "$LATEST_JAR" "$current_class" | head -2 | tail -1)
|
||||
if [[ $class_definition == *"interface"* ]]
|
||||
then
|
||||
LATEST_METHODS=${LATEST_METHODS//abstract /}
|
||||
CURRENT_METHODS=${CURRENT_METHODS//abstract /}
|
||||
else
|
||||
LATEST_METHODS=${LATEST_METHODS//synchronized /}
|
||||
CURRENT_METHODS=${CURRENT_METHODS//synchronized /}
|
||||
fi
|
||||
}
|
||||
|
||||
# Checks if there are any methods in the latest version that were removed in the current version.
|
||||
find_removed_methods() {
|
||||
echo "Checking if methods in current version (v$CURRENT_VERSION) were removed from latest version (v$LATEST_VERSION)"
|
||||
if is_new_minor_release || is_new_major_release
|
||||
then
|
||||
echo "New minor/major release is being performed. Ignoring changes in classes marked with @KinesisClientInternalApi annotation."
|
||||
fi
|
||||
local latest_classes=$(
|
||||
jar tf $LATEST_JAR |
|
||||
grep .class |
|
||||
tr / . |
|
||||
sed 's/\.class$//' |
|
||||
# skip generated proto classes since these have a lot of inherited methods
|
||||
# that are not outputted by javap. besides, generated java code is not a
|
||||
# good indicator of proto compatibility- it will not capture reserved
|
||||
# tags or deprecated fields.
|
||||
grep -v 'software\.amazon\.kinesis\.retrieval\.kpl\.Messages')
|
||||
for class in $latest_classes
|
||||
do
|
||||
if is_kinesis_client_internal_api "$class" || is_non_public_class "$class"
|
||||
then
|
||||
continue
|
||||
fi
|
||||
|
||||
CURRENT_METHODS=$(javap -classpath "$CURRENT_JAR" "$class" 2>/dev/null)
|
||||
if [ -z "$CURRENT_METHODS" ]
|
||||
then
|
||||
echo "Class $class was removed"
|
||||
REMOVED_METHODS_FLAG=$TRUE
|
||||
continue
|
||||
fi
|
||||
|
||||
LATEST_METHODS=$(javap -classpath "$LATEST_JAR" "$class")
|
||||
|
||||
ignore_non_breaking_changes "$class"
|
||||
|
||||
local removed_methods=$(diff <(echo "$LATEST_METHODS") <(echo "$CURRENT_METHODS") | grep '^<')
|
||||
|
||||
# ignore synthetic access methods - these are not available to users and will not break backwards compatibility
|
||||
removed_methods=$(echo "$removed_methods" | grep -v "access\$[0-9]\+")
|
||||
|
||||
if [[ "$removed_methods" != "" ]]
|
||||
then
|
||||
REMOVED_METHODS_FLAG=$TRUE
|
||||
echo "$class does not have method(s):"
|
||||
echo "$removed_methods"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
get_backwards_compatible_result() {
|
||||
if [[ $REMOVED_METHODS_FLAG == $TRUE ]]
|
||||
then
|
||||
echo "Current KCL version $CURRENT_VERSION is not backwards compatible with version $LATEST_VERSION. See output above for removed packages/methods."
|
||||
is_new_major_release || exit 1
|
||||
else
|
||||
echo "Current KCL version $CURRENT_VERSION is backwards compatible with version $LATEST_VERSION."
|
||||
exit 0
|
||||
fi
|
||||
}
|
||||
|
||||
main() {
|
||||
get_latest_jar
|
||||
get_current_jar
|
||||
find_removed_methods
|
||||
get_backwards_compatible_result
|
||||
}
|
||||
|
||||
main
|
||||
43
.github/workflows/maven.yml
vendored
43
.github/workflows/maven.yml
vendored
|
|
@ -7,26 +7,59 @@
|
|||
# documentation.
|
||||
|
||||
name: Java CI with Maven
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- "master"
|
||||
- "v2.x"
|
||||
- "v1.x"
|
||||
pull_request:
|
||||
branches:
|
||||
- "master"
|
||||
- "v2.x"
|
||||
- "v1.x"
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
build:
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/checkout@v4
|
||||
- name: Set up JDK 8
|
||||
uses: actions/setup-java@v3
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: '8'
|
||||
distribution: 'corretto'
|
||||
- name: Build with Maven
|
||||
run: mvn -B package --file pom.xml -DskipITs
|
||||
backwards-compatible-check:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Set up JDK 8
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: '8'
|
||||
distribution: 'corretto'
|
||||
- name: Check backwards compatibility of changes
|
||||
run: .github/scripts/backwards_compatibility_check.sh
|
||||
auto-merge:
|
||||
needs: [build]
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event.pull_request.user.login == 'dependabot[bot]'
|
||||
steps:
|
||||
- name: Dependabot metadata
|
||||
id: metadata
|
||||
uses: dependabot/fetch-metadata@v2
|
||||
with:
|
||||
alert-lookup: true
|
||||
github-token: "${{ secrets.GITHUB_TOKEN }}"
|
||||
- name: Enable auto-merge for Dependabot PRs
|
||||
if: steps.metadata.outputs.update-type == 'version-update:semver-patch' && steps.metadata.outputs.cvss > 0
|
||||
run: gh pr merge --auto --merge "$PR_URL"
|
||||
env:
|
||||
PR_URL: ${{github.event.pull_request.html_url}}
|
||||
GH_TOKEN: ${{secrets.GITHUB_TOKEN}}
|
||||
618
CHANGELOG.md
618
CHANGELOG.md
|
|
@ -2,563 +2,63 @@
|
|||
|
||||
For **1.x** release notes, please see [v1.x/CHANGELOG.md](https://github.com/awslabs/amazon-kinesis-client/blob/v1.x/CHANGELOG.md)
|
||||
|
||||
For **2.x** release notes, please see [v2.x/CHANGELOG.md](https://github.com/awslabs/amazon-kinesis-client/blob/v2.x/CHANGELOG.md)
|
||||
|
||||
---
|
||||
### Release 2.5.1 (June 27, 2023)
|
||||
* [#1143](https://github.com/awslabs/amazon-kinesis-client/pull/1143) Upgrade MultiLangDaemon to support StreamARN
|
||||
* [#1145](https://github.com/awslabs/amazon-kinesis-client/pull/1145) Introduced GitHub actions to trigger Maven builds during merge/pull requests
|
||||
* [#1136](https://github.com/awslabs/amazon-kinesis-client/pull/1136) Added testing architecture and KCL 2.x basic polling/streaming tests
|
||||
* [#1153](https://github.com/awslabs/amazon-kinesis-client/pull/1153) Checkstyle: added `UnusedImports` check.
|
||||
* [#1150](https://github.com/awslabs/amazon-kinesis-client/pull/1150) Enabled Checkstyle validation of test resources.
|
||||
* [#1149](https://github.com/awslabs/amazon-kinesis-client/pull/1149) Bound Checkstyle to `validate` goal for automated enforcement.
|
||||
* [#1148](https://github.com/awslabs/amazon-kinesis-client/pull/1148) Code cleanup to faciliate Checkstyle enforcement.
|
||||
* [#1142](https://github.com/awslabs/amazon-kinesis-client/pull/1142) Upgrade Google Guava dependency version from 31.1-jre to 32.0.0-jre
|
||||
* [#1115](https://github.com/awslabs/amazon-kinesis-client/pull/1115) Update KCL version from 2.5.0 to 2.5.1-SNAPSHOT
|
||||
|
||||
### Release 2.5.0 (May 19, 2023)
|
||||
* **[#1109](https://github.com/awslabs/amazon-kinesis-client/pull/1109) Add support for stream ARNs**
|
||||
* **[#1065](https://github.com/awslabs/amazon-kinesis-client/pull/1065) Allow tags to be added when lease table is created**
|
||||
* [#1094](https://github.com/awslabs/amazon-kinesis-client/pull/1094) Code cleanup to introduce better testing
|
||||
* [#1088](https://github.com/awslabs/amazon-kinesis-client/pull/1088) Minimize race in PSSM to optimize shard sync calls
|
||||
* [#1086](https://github.com/awslabs/amazon-kinesis-client/pull/1086) Add additional SingleStreamTracker constructor with stream position parameter
|
||||
* [#1084](https://github.com/awslabs/amazon-kinesis-client/pull/1084) More consistent testing behavior with restartAfterRequestTimerExpires
|
||||
* [#1066](https://github.com/awslabs/amazon-kinesis-client/pull/1066) More consistent testing behavior with HashRangesAreAlwaysComplete
|
||||
* [#1072](https://github.com/awslabs/amazon-kinesis-client/pull/1072) Upgrade nexus-staging-maven-plugin from 1.6.8 to 1.6.13
|
||||
* [#1073](https://github.com/awslabs/amazon-kinesis-client/pull/1073) Upgrade slf4j-api from 2.0.6 to 2.0.7
|
||||
* [#1090](https://github.com/awslabs/amazon-kinesis-client/pull/1090) Upgrade awssdk.version from 2.20.8 to 2.20.43
|
||||
* [#1071](https://github.com/awslabs/amazon-kinesis-client/pull/1071) Upgrade maven-compiler-plugin from 3.8.1 to 3.11.0
|
||||
|
||||
### Release 2.4.8 (March 21, 2023)
|
||||
* [#1080](https://github.com/awslabs/amazon-kinesis-client/pull/1080) Added metric in `ShutdownTask` for scenario when parent leases are missing.
|
||||
* [#1077](https://github.com/awslabs/amazon-kinesis-client/pull/1077) Reverted changes to pom property
|
||||
* [#1069](https://github.com/awslabs/amazon-kinesis-client/pull/1069) Fixed flaky InitializationWaitsWhenLeaseTableIsEmpty test
|
||||
|
||||
|
||||
### Release 2.4.7 (March 17, 2023)
|
||||
* **NOTE: Due to an issue during the release process, the 2.4.7 published artifacts are incomplete and non-viable. Please use 2.4.8 or later.**
|
||||
* [#1063](https://github.com/awslabs/amazon-kinesis-client/pull/1063) Allow leader to learn new leases upon re-election to avoid unnecessary shardSyncs
|
||||
* [#1060](https://github.com/awslabs/amazon-kinesis-client/pull/1060) Add new metric to be emitted on lease creation
|
||||
* [#1057](https://github.com/awslabs/amazon-kinesis-client/pull/1057) Added more logging in `Scheduler` w.r.t. `StreamConfig`s.
|
||||
* [#1059](https://github.com/awslabs/amazon-kinesis-client/pull/1059) DRY: simplification of `HierarchicalShardSyncerTest`.
|
||||
* [#1062](https://github.com/awslabs/amazon-kinesis-client/pull/1062) Fixed retry storm in `PrefetchRecordsPublisher`.
|
||||
* [#1061](https://github.com/awslabs/amazon-kinesis-client/pull/1061) Fixed NPE in `LeaseCleanupManager`.
|
||||
* [#1056](https://github.com/awslabs/amazon-kinesis-client/pull/1056) Clean up in-memory state of deleted kinesis stream in MultiStreamMode
|
||||
* [#1058](https://github.com/awslabs/amazon-kinesis-client/pull/1058) Documentation: added `<pre>` tags so fixed-format diagrams aren't garbled.
|
||||
* [#1053](https://github.com/awslabs/amazon-kinesis-client/pull/1053) Exposed convenience method of `ExtendedSequenceNumber#isSentinelCheckpoint()`
|
||||
* [#1043](https://github.com/awslabs/amazon-kinesis-client/pull/1043) Removed a `.swp` file, and updated `.gitignore`.
|
||||
* [#1047](https://github.com/awslabs/amazon-kinesis-client/pull/1047) Upgrade awssdk.version from 2.19.31 to 2.20.8
|
||||
* [#1046](https://github.com/awslabs/amazon-kinesis-client/pull/1046) Upgrade maven-javadoc-plugin from 3.3.1 to 3.5.0
|
||||
* [#1038](https://github.com/awslabs/amazon-kinesis-client/pull/1038) Upgrade gsr.version from 1.1.13 to 1.1.14
|
||||
* [#1037](https://github.com/awslabs/amazon-kinesis-client/pull/1037) Upgrade aws-java-sdk.version from 1.12.370 to 1.12.405
|
||||
|
||||
### Release 2.4.6 (February 21, 2023)
|
||||
* [#1041](https://github.com/awslabs/amazon-kinesis-client/pull/1041) Minor optimizations (e.g., calculate-once, put instead of get+put)
|
||||
* [#1035](https://github.com/awslabs/amazon-kinesis-client/pull/1035) Release Note updates to avoid duplication and bitrot (e.g., 1.x release
|
||||
* [#935](https://github.com/awslabs/amazon-kinesis-client/pull/935) Pass isAtShardEnd correctly to processRecords call
|
||||
* [#1040](https://github.com/awslabs/amazon-kinesis-client/pull/1040) Increased logging verbosity around lease management
|
||||
* [#1024](https://github.com/awslabs/amazon-kinesis-client/pull/1024) Added logging w.r.t. StreamConfig handling.
|
||||
* [#1034](https://github.com/awslabs/amazon-kinesis-client/pull/1034) Optimization: 9~15% improvement in KinesisDataFetcher wall-time
|
||||
* [#1045](https://github.com/awslabs/amazon-kinesis-client/pull/1045) Fixed duplication of project version in children pom.xml
|
||||
* [#956](https://github.com/awslabs/amazon-kinesis-client/pull/956) Fixed warning message typos
|
||||
* [#795](https://github.com/awslabs/amazon-kinesis-client/pull/795) Fixed log message spacing
|
||||
* [#740](https://github.com/awslabs/amazon-kinesis-client/pull/740) Fixed typo in Comment
|
||||
* [#1028](https://github.com/awslabs/amazon-kinesis-client/pull/1028) Refactored MultiStreamTracker to provide and enhance OOP for both
|
||||
* [#1027](https://github.com/awslabs/amazon-kinesis-client/pull/1027) Removed CHECKSTYLE:OFF toggles which can invite/obscure sub-par code.
|
||||
* [#1032](https://github.com/awslabs/amazon-kinesis-client/pull/1032) Upgrade rxjava from 3.1.5 to 3.1.6
|
||||
* [#1030](https://github.com/awslabs/amazon-kinesis-client/pull/1030) Upgrade awssdk.version from 2.19.2 to 2.19.31
|
||||
* [#1029](https://github.com/awslabs/amazon-kinesis-client/pull/1029) Upgrade slf4j-api from 2.0.0 to 2.0.6
|
||||
* [#1015](https://github.com/awslabs/amazon-kinesis-client/pull/1015) Upgrade protobuf-java from 3.21.5 to 3.21.12
|
||||
|
||||
### Release 2.4.5 (January 04, 2023)
|
||||
* [#1014](https://github.com/awslabs/amazon-kinesis-client/pull/1014) Use AFTER_SEQUENCE_NUMBER iterator type for expired iterator request
|
||||
|
||||
### Release 2.4.4 (December 23, 2022)
|
||||
* [#1017](https://github.com/awslabs/amazon-kinesis-client/pull/1017) Upgrade aws sdk
|
||||
* aws-java-sdk.version from 1.12.296 -> 1.12.370
|
||||
* awssdk.version from 2.17.268 -> 2.19.2
|
||||
* [#1020](https://github.com/awslabs/amazon-kinesis-client/pull/1020) Correct the KCL version in the main pom
|
||||
|
||||
### Release 2.4.3 (September 6, 2022)
|
||||
* [#980](https://github.com/awslabs/amazon-kinesis-client/pull/980) logback-classic: 1.2.9 -> 1.4.0
|
||||
* [#983](https://github.com/awslabs/amazon-kinesis-client/pull/983)
|
||||
* protobuf-java: 3.19.2 -> 3.21.5
|
||||
* slf4j.version: 1.7.32 -> 2.0.0
|
||||
* schema-registry-serde: 1.1.9 -> 1.1.13
|
||||
* [#984](https://github.com/awslabs/amazon-kinesis-client/pull/984) awssdk.version from 2.17.108 to 2.17.267
|
||||
* [#987](https://github.com/awslabs/amazon-kinesis-client/pull/987) guava: 31.0.1-jre -> 31.1-jre
|
||||
* [#988](https://github.com/awslabs/amazon-kinesis-client/pull/988) jcommander: 1.81 to 1.82
|
||||
* [#990](https://github.com/awslabs/amazon-kinesis-client/pull/990) Upgrade dependencies
|
||||
* aws-java-sdk.version: 1.12.130 -> 1.12.296
|
||||
* lombok: 1.18.22 -> 1.18.24
|
||||
* rxjava: 3.1.3 -> 3.1.5
|
||||
* maven-resources-plugin: 2.6 -> 3.3.0
|
||||
* logback-classic: 1.4.0 -> 1.3.0
|
||||
* awssdk.version: 2.17.267 -> 2.17.268
|
||||
|
||||
### Release 2.4.2 (August 10, 2022)
|
||||
* [#972](https://github.com/awslabs/amazon-kinesis-client/pull/972) Upgrade Lombok to version 1.18.24
|
||||
|
||||
### Latest Release 2.4.1 (March 24, 2022)
|
||||
[Milestone#68](https://github.com/awslabs/amazon-kinesis-client/milestone/68)
|
||||
* [#916](https://github.com/awslabs/amazon-kinesis-client/pull/916) Upgrade to rxjava3
|
||||
|
||||
### Release 2.4.0 (March 2, 2022)
|
||||
[Milestone#67](https://github.com/awslabs/amazon-kinesis-client/milestone/67)
|
||||
* [#894](https://github.com/awslabs/amazon-kinesis-client/pull/894) Bump protobuf-java from 3.19.1 to 3.19.2
|
||||
* [#924](https://github.com/awslabs/amazon-kinesis-client/pull/924) Support Protobuf Data format with Glue Schema Registry.
|
||||
|
||||
### Latest Release 2.3.10 (January 4, 2022)
|
||||
[Milestone#66](https://github.com/awslabs/amazon-kinesis-client/milestone/66)
|
||||
* [#868](https://github.com/awslabs/amazon-kinesis-client/pull/868) Adding a new metric: Application-level MillisBehindLatest
|
||||
* [#879](https://github.com/awslabs/amazon-kinesis-client/pull/879) Keep dependencies up-to-date
|
||||
* [#886](https://github.com/awslabs/amazon-kinesis-client/pull/886) Get latest counter before attempting a take to ensure take succeeds
|
||||
* [#888](https://github.com/awslabs/amazon-kinesis-client/pull/888) Configure dependabot for v1.x branch
|
||||
|
||||
### Release 2.3.9 (November 22, 2021)
|
||||
[Milestone#65](https://github.com/awslabs/amazon-kinesis-client/milestone/65)
|
||||
* [#866](https://github.com/awslabs/amazon-kinesis-client/pull/866) Update logback dependency.
|
||||
|
||||
### Release 2.3.8 (October 27, 2021)
|
||||
[Milestone#64](https://github.com/awslabs/amazon-kinesis-client/milestone/64)
|
||||
* [#860](https://github.com/awslabs/amazon-kinesis-client/pull/860) Upgrade Glue schema registry from 1.1.4 to 1.1.5.
|
||||
* [#861](https://github.com/awslabs/amazon-kinesis-client/pull/861) Revert [PR#847](https://github.com/awslabs/amazon-kinesis-client/pull/847) due to regression for leases without owners and added new tests.
|
||||
|
||||
### Release 2.3.7 (October 11, 2021)
|
||||
[Milestone#63](https://github.com/awslabs/amazon-kinesis-client/milestone/63)
|
||||
* [#842](https://github.com/awslabs/amazon-kinesis-client/pull/842) Fixing typo is debug logs.
|
||||
* [#846](https://github.com/awslabs/amazon-kinesis-client/pull/846) Fix DynamoDBLeaseTaker logging of available leases
|
||||
* [#847](https://github.com/awslabs/amazon-kinesis-client/pull/847) Make use of Java 8 to simplify computeLeaseCounts()
|
||||
* [#853](https://github.com/awslabs/amazon-kinesis-client/pull/853) Add configurable initial position for orphaned stream
|
||||
* [#854](https://github.com/awslabs/amazon-kinesis-client/pull/854) Create DynamoDB tables on On-Demand billing mode by default.
|
||||
* [#855](https://github.com/awslabs/amazon-kinesis-client/pull/855) Emit Glue Schema Registry usage metrics
|
||||
* [#857](https://github.com/awslabs/amazon-kinesis-client/pull/857) Fix to shutdown PrefetchRecordsPublisher in gracefull manner
|
||||
* [#858](https://github.com/awslabs/amazon-kinesis-client/pull/858) Upgrade AWS SDK version to 2.17.52.
|
||||
|
||||
### Release 2.3.6 (July 9, 2021)
|
||||
[Milestone#62](https://github.com/awslabs/amazon-kinesis-client/milestone/62)
|
||||
* [#836](https://github.com/awslabs/amazon-kinesis-client/pull/836) Upgraded AWS SDK version to 2.16.98
|
||||
* [#835](https://github.com/awslabs/amazon-kinesis-client/pull/835) Upgraded Glue Schema Registry version to 1.1.1
|
||||
* [#828](https://github.com/awslabs/amazon-kinesis-client/pull/828) Modified wildcard imports to individual imports.
|
||||
* [#817](https://github.com/awslabs/amazon-kinesis-client/pull/817) Updated the Worker shutdown logic to make sure that the `LeaseCleanupManager` also terminates all the threads that it has started.
|
||||
* [#794](https://github.com/awslabs/amazon-kinesis-client/pull/794) Silence warning when there are no stale streams to delete.
|
||||
|
||||
### Release 2.3.5 (June 14, 2021)
|
||||
[Milestone#59](https://github.com/awslabs/amazon-kinesis-client/milestone/59)
|
||||
* [#824](https://github.com/awslabs/amazon-kinesis-client/pull/824) Upgraded dependencies
|
||||
* logback-classic version to 1.2.3
|
||||
* AWS Java SDK version to 1.12.3
|
||||
* AWS SDK version to 2.16.81
|
||||
* [#815](https://github.com/awslabs/amazon-kinesis-client/pull/815) Converted Future to CompletableFuture which helps in proper conversion to Scala using Scala Future Converters.
|
||||
* [#810](https://github.com/awslabs/amazon-kinesis-client/pull/810) Bump commons-io from 2.6 to 2.7
|
||||
* [#804](https://github.com/awslabs/amazon-kinesis-client/pull/804) Allowing user to specify an initial timestamp in which daemon will process records.
|
||||
* [#802](https://github.com/awslabs/amazon-kinesis-client/pull/802) Upgraded guava from 26.0-jre to 29.0-jre
|
||||
* [#801](https://github.com/awslabs/amazon-kinesis-client/pull/801) Fixing a bug that causes to block indefinitely when trying to unlock a lock that isn't locked.
|
||||
* [#762](https://github.com/awslabs/amazon-kinesis-client/pull/762) Added support for web identity token in multilang
|
||||
|
||||
### Release 2.3.4 (February 19, 2021)
|
||||
[Milestone#56](https://github.com/awslabs/amazon-kinesis-client/milestone/56)
|
||||
* [#788](https://github.com/awslabs/amazon-kinesis-client/pull/788) Fixing a bug that caused paginated `ListShards` calls with the `ShardFilter` parameter to fail when the lease table was being initialized.
|
||||
|
||||
### Release 2.3.3 (December 23, 2020)
|
||||
[Milestone#55](https://github.com/awslabs/amazon-kinesis-client/milestone/55)
|
||||
* Fixing bug in PrefetchRecordsPublisher which was causing retry storms if initial request fails.
|
||||
* Fixing bug where idleTimeBetweenReadsInMillis property was ignored in PollingConfig.
|
||||
|
||||
### Release 2.3.2 (November 19, 2020)
|
||||
[Milestone#54](https://github.com/awslabs/amazon-kinesis-client/milestone/54)
|
||||
* Adding support for Glue Schema Registry. Deserialize and read schemas associated with the records.
|
||||
* Updating AWS SDK version to 2.15.31
|
||||
|
||||
|
||||
### Release 2.3.1 (October 20, 2020)
|
||||
[Milestone#53](https://github.com/awslabs/amazon-kinesis-client/milestone/53)
|
||||
* Introducing support for processing multiple kinesis data streams with the same KCL 2.x for java consumer application
|
||||
* To build a consumer application that can process multiple streams at the same time, you must implement a new
|
||||
interface called MultistreamTracker (https://github.com/awslabs/amazon-kinesis-client/blob/0c5042dadf794fe988438436252a5a8fe70b6b0b/amazon-kinesis-client/src/main/java/software/amazon/kinesis/processor/MultiStreamTracker.java)
|
||||
|
||||
* MultistreamTracker will also publish various metrics around the current active streams being processed, the number
|
||||
of streams which are deleted at this time period or are pending deletion.
|
||||
|
||||
|
||||
### Release 2.3.0 (August 17, 2020)
|
||||
* [Milestone#52](https://github.com/awslabs/amazon-kinesis-client/milestones/52)
|
||||
|
||||
* Behavior of shard synchronization is moving from each worker independently learning about all existing shards to workers only discovering the children of shards that each worker owns. This optimizes memory usage, lease table IOPS usage, and number of calls made to kinesis for streams with high shard counts and/or frequent resharding.
|
||||
* When bootstrapping an empty lease table, KCL utilizes the `ListShard` API's filtering option (the ShardFilter optional request parameter) to retrieve and create leases only for a snapshot of shards open at the time specified by the `ShardFilter` parameter. The `ShardFilter` parameter enables you to filter out the response of the `ListShards` API, using the `Type` parameter. KCL uses the `Type` filter parameter and the following of its valid values to identify and return a snapshot of open shards that might require new leases.
|
||||
* Currently, the following shard filters are supported:
|
||||
* `AT_TRIM_HORIZON` - the response includes all the shards that were open at `TRIM_HORIZON`.
|
||||
* `AT_LATEST` - the response includes only the currently open shards of the data stream.
|
||||
* `AT_TIMESTAMP` - the response includes all shards whose start timestamp is less than or equal to the given timestamp and end timestamp is greater than or equal to the given timestamp or still open.
|
||||
* `ShardFilter` is used when creating leases for an empty lease table to initialize leases for a snapshot of shards specified at `RetrievalConfig#initialPositionInStreamExtended`.
|
||||
* For more information about ShardFilter, see the [official AWS documentation on ShardFilter](https://docs.aws.amazon.com/kinesis/latest/APIReference/API_ShardFilter.html).
|
||||
|
||||
* Introducing support for the `ChildShards` response of the `GetRecords` and the `SubscribeToShard` APIs to perform lease/shard synchronization that happens at `SHARD_END` for closed shards, allowing a KCL worker to only create leases for the child shards of the shard it finished processing.
|
||||
* For shared throughout consumer applications, this uses the `ChildShards` response of the `GetRecords` API. For dedicated throughput (enhanced fan-out) consumer applications, this uses the `ChildShards` response of the `SubscribeToShard` API.
|
||||
* For more information, see the official AWS Documentation on [GetRecords](https://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetRecords.html), [SubscribeToShard](https://docs.aws.amazon.com/kinesis/latest/APIReference/API_SubscribeToShard.html), and [ChildShard](https://docs.aws.amazon.com/kinesis/latest/APIReference/API_ChildShard.html).
|
||||
|
||||
* KCL now also performs additional periodic shard/lease scans in order to identify any potential holes in the lease table to ensure the complete hash range of the stream is being processed and create leases for them if required. `PeriodicShardSyncManager` is the new component that is responsible for running periodic lease/shard scans.
|
||||
* New configuration options are available to configure `PeriodicShardSyncManager` in `LeaseManagementConfig`
|
||||
|
||||
| Name | Default | Description |
|
||||
| ----------------------------------------------------- | ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| leasesRecoveryAuditorExecutionFrequencyMillis | 120000 (2 minutes) | Frequency (in millis) of the auditor job to scan for partial leases in the lease table. If the auditor detects any hole in the leases for a stream, then it would trigger shard sync based on leasesRecoveryAuditorInconsistencyConfidenceThreshold. |
|
||||
| leasesRecoveryAuditorInconsistencyConfidenceThreshold | 3 | Confidence threshold for the periodic auditor job to determine if leases for a stream in the lease table is inconsistent. If the auditor finds same set of inconsistencies consecutively for a stream for this many times, then it would trigger a shard sync |
|
||||
|
||||
* New CloudWatch metrics are also now emitted to monitor the health of `PeriodicShardSyncManager`:
|
||||
|
||||
| Name | Description |
|
||||
| --------------------------- | ------------------------------------------------------ |
|
||||
| NumStreamsWithPartialLeases | Number of streams that had holes in their hash ranges. |
|
||||
| NumStreamsToSync | Number of streams which underwent a full shard sync. |
|
||||
|
||||
* Introducing deferred lease cleanup. Leases will be deleted asynchronously by `LeaseCleanupManager` upon reaching `SHARD_END`, when a shard has either expired past the stream’s retention period or been closed as the result of a resharding operation.
|
||||
* New configuration options are available to configure `LeaseCleanupManager`.
|
||||
|
||||
| Name | Default | Description |
|
||||
| ----------------------------------- | ---------- | --------------------------------------------------------------------------------------------------------- |
|
||||
| leaseCleanupIntervalMillis | 1 minute | Interval at which to run lease cleanup thread. |
|
||||
| completedLeaseCleanupIntervalMillis | 5 minutes | Interval at which to check if a lease is completed or not. |
|
||||
| garbageLeaseCleanupIntervalMillis | 30 minutes | Interval at which to check if a lease is garbage (i.e trimmed past the stream's retention period) or not. |
|
||||
|
||||
* Introducing _experimental_ support for multistreaming, allowing a single KCL application to multiplex processing multiple streams.
|
||||
* New configuration options are available to enable multistreaming in `RetrievalConfig#appStreamTracker`.
|
||||
|
||||
* Fixing a bug in `PrefetchRecordsPublisher` restarting while it was already running.
|
||||
* Including an optimization to `HierarchicalShardSyncer` to only create leases for one layer of shards.
|
||||
* Adding support to prepare and commit lease checkpoints with arbitrary bytes.
|
||||
* This allows checkpointing of an arbitrary byte buffer up to the maximum permitted DynamoDB item size ([currently 400 KB as of release](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Limits.html)), and can be used for recovery by passing a serialized byte buffer to `RecordProcessorCheckpointer#prepareCheckpoint` and `RecordProcessorCheckpointer#checkpoint`.
|
||||
* Upgrading version of AWS SDK to 2.14.0.
|
||||
* [#725](https://github.com/awslabs/amazon-kinesis-client/pull/725) Allowing KCL to consider lease tables in `UPDATING` healthy.
|
||||
|
||||
### Release 2.2.11 (May 28, 2020)
|
||||
[Milestone#51](https://github.com/awslabs/amazon-kinesis-client/milestone/51)
|
||||
* Adjusting HTTP2 initial window size to 512 KB
|
||||
* [PR#706](https://github.com/awslabs/amazon-kinesis-client/pull/706)
|
||||
* Updating protobuf-java to version 3.11.4
|
||||
* [PR#718](https://github.com/awslabs/amazon-kinesis-client/pull/718)
|
||||
* Updating the AWS Java SDK to version 2.13.25
|
||||
* [PR#722](https://github.com/awslabs/amazon-kinesis-client/pull/722)
|
||||
|
||||
### Release 2.2.10 (March 26, 2020)
|
||||
[Milestone#48](https://github.com/awslabs/amazon-kinesis-client/milestone/48)
|
||||
* Fixing a bug in DynamoDB billing mode support for special regions.
|
||||
* [PR#703](https://github.com/awslabs/amazon-kinesis-client/pull/703)
|
||||
* Adding request id logging to ShardConsumerSubscriber.
|
||||
* [PR#705](https://github.com/awslabs/amazon-kinesis-client/pull/705)
|
||||
|
||||
### Release 2.2.9 (Febuary 17, 2020)
|
||||
[Milestone#47](https://github.com/awslabs/amazon-kinesis-client/milestone/47)
|
||||
* Updating the AWS SDK version to 2.10.66.
|
||||
* [PR#687](https://github.com/awslabs/amazon-kinesis-client/commit/8aaf2aa11c43f77f459732cdb7d88f4418d367ff)
|
||||
* Adding request id logging to SubscribeToShard response.
|
||||
* [PR#678](https://github.com/awslabs/amazon-kinesis-client/pull/678)
|
||||
|
||||
### Release 2.2.8 (January 28, 2020)
|
||||
[Milestone#46](https://github.com/awslabs/amazon-kinesis-client/milestone/45)
|
||||
* Updating the AWS SDK version to 2.10.56.
|
||||
* [PR#679](https://github.com/awslabs/amazon-kinesis-client/pull/679)
|
||||
* NOTE: SDK has a known connection teardown issue when multiple H2 streams are used within a connection. This might result in shard consumers sticking to a stale service host and not progressing. If your shard consumer gets stuck, use the following configuration as a workaround. This configuration might result in up to 5X increase in total connections.
|
||||
```
|
||||
KinesisAsyncClient kinesisClient = KinesisAsyncClient.builder()
|
||||
.region(region)
|
||||
.httpClientBuilder(NettyNioAsyncHttpClient.builder().maxConcurrency(Integer.MAX_VALUE).http2Configuration(Http2Configuration.builder().maxStreams(1).build())
|
||||
.build()
|
||||
```
|
||||
* Making ShardConsumerTest resilient to race conditions.
|
||||
* [PR#668](https://github.com/awslabs/amazon-kinesis-client/pull/668)
|
||||
* Updating integration test naming.
|
||||
* [PR#667](https://github.com/awslabs/amazon-kinesis-client/pull/667)
|
||||
|
||||
### Release 2.2.7 (December 2, 2019)
|
||||
[Milestone#45](https://github.com/awslabs/amazon-kinesis-client/milestone/45)
|
||||
* Updating the AWS SDK version to 2.10.25
|
||||
* [PR#657](https://github.com/awslabs/amazon-kinesis-client/pull/657)
|
||||
* Adding a configurable DynamoDB billing mode
|
||||
* [PR#582](https://github.com/awslabs/amazon-kinesis-client/pull/582)
|
||||
* NOTE: Billing mode is not available in all regions; if your lease table cannot be created, use the following configuration as a workaround:
|
||||
```
|
||||
LeaseManagementConfig leaseManagementConfig = builder.leaseManagementConfig().billingMode(null).build();
|
||||
```
|
||||
|
||||
|
||||
### Release 2.2.6 (November 7, 2019)
|
||||
[Milestone#43](https://github.com/awslabs/amazon-kinesis-client/milestone/43)
|
||||
* Updating the SDK version to 2.9.25.
|
||||
* [PR#638](https://github.com/awslabs/amazon-kinesis-client/pull/638)
|
||||
* Clearing the local cache on a subscription termination, to avoid noisy logs on new subscriptions.
|
||||
* [PR#642](https://github.com/awslabs/amazon-kinesis-client/pull/642)
|
||||
* Updating the SDK version to 2.10.0 in order to fix the premature H2 stream close issue.
|
||||
* [PR#649](https://github.com/awslabs/amazon-kinesis-client/pull/649)
|
||||
* NOTE: SDK has a known connection teardown issue when multiple H2 streams are used within a connection. This might result in shard consumers sticking to a stale service host and not progressing. If your shard consumer gets stuck, use the following configuration as a workaround. This configuration might result in up to 5X increase in total connections.
|
||||
```
|
||||
KinesisAsyncClient kinesisClient = KinesisAsyncClient.builder()
|
||||
.region(region)
|
||||
.httpClientBuilder(NettyNioAsyncHttpClient.builder().maxConcurrency(Integer.MAX_VALUE).maxHttp2Streams(1))
|
||||
.build()
|
||||
```
|
||||
|
||||
### Release 2.2.5 (October 23, 2019)
|
||||
|
||||
[Milestone#40](https://github.com/awslabs/amazon-kinesis-client/milestone/40)
|
||||
* Updating Sonatype to dedicated AWS endpoint.
|
||||
* [PR#619](https://github.com/awslabs/amazon-kinesis-client/pull/619)
|
||||
* Introducing a validation step to verify if ShardEnd is reached, to prevent shard consumer stuck scenarios in the event of malformed response from service.
|
||||
* [PR#624](https://github.com/awslabs/amazon-kinesis-client/pull/624)
|
||||
|
||||
### Release 2.2.4 (September 23, 2019)
|
||||
|
||||
[Milestone#39](https://github.com/awslabs/amazon-kinesis-client/milestone/39)
|
||||
* Making FanoutRecordsPublisher test cases resilient to delayed thread operations
|
||||
* [PR#612](https://github.com/awslabs/amazon-kinesis-client/pull/612)
|
||||
* Drain delivery queue in the FanoutRecordsPublisher to make slow consumers consume events at their pace
|
||||
* [PR#607](https://github.com/awslabs/amazon-kinesis-client/pull/607)
|
||||
* Fix to prevent the onNext event going to stale subscription when restart happens in PrefetchRecordsPublisher
|
||||
* [PR#606](https://github.com/awslabs/amazon-kinesis-client/pull/606)
|
||||
|
||||
### Release 2.2.3 (September 04, 2019)
|
||||
|
||||
[Milestone#38](https://github.com/awslabs/amazon-kinesis-client/milestone/38)
|
||||
* Fix to prevent data loss and stuck shards in the event of failed records delivery in Polling readers
|
||||
* [PR#603](https://github.com/awslabs/amazon-kinesis-client/pull/603)
|
||||
|
||||
### Release 2.2.2 (August 19, 2019)
|
||||
|
||||
[Milestone#36](https://github.com/awslabs/amazon-kinesis-client/milestone/36)
|
||||
* Fix to prevent invalid ShardConsumer state transitions due to rejected executor service executions.
|
||||
* [PR#560](https://github.com/awslabs/amazon-kinesis-client/pull/560)
|
||||
* Fixing a bug in which initial subscription failure caused a shard consumer to get stuck.
|
||||
* [PR#562](https://github.com/awslabs/amazon-kinesis-client/pull/562)
|
||||
* Making CW publish failures visible by executing the async publish calls in a blocking manner and logging on exception.
|
||||
* [PR#584](https://github.com/awslabs/amazon-kinesis-client/pull/584)
|
||||
* Update shard end checkpoint failure messaging.
|
||||
* [PR#591](https://github.com/awslabs/amazon-kinesis-client/pull/591)
|
||||
* A fix for resiliency and durability issues that occur in the reduced thread mode - Nonblocking approach.
|
||||
* [PR#573](https://github.com/awslabs/amazon-kinesis-client/pull/573)
|
||||
* Preventing duplicate delivery due to unacknowledged event, while completing the subscription.
|
||||
* [PR#596](https://github.com/awslabs/amazon-kinesis-client/pull/596)
|
||||
|
||||
### Release 2.2.1 (July 1, 2019)
|
||||
[Milestone#32](https://github.com/awslabs/amazon-kinesis-client/milestone/32)
|
||||
* Add periodic logging for the state of the thread pool executor service. This service executes the async tasks submitted to and by the ShardConsumer.
|
||||
* Add logging of failures from RxJava layer.
|
||||
* [PR#559](https://github.com/awslabs/amazon-kinesis-client/pull/559)
|
||||
|
||||
### Release 2.2.0 (April 8, 2019)
|
||||
[Milestone#31](https://github.com/awslabs/amazon-kinesis-client/milestone/31)
|
||||
* Updated License to [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0)
|
||||
* [PR#523](https://github.com/awslabs/amazon-kinesis-client/pull/523)
|
||||
* Introducing configuration for suppressing logs from ReadTimeoutExceptions caused while calling SubscribeToShard.
|
||||
Suppression can be configured by setting `LifecycleConfig#readTimeoutsToIgnoreBeforeWarning(Count)`.
|
||||
* [PR#528](https://github.com/awslabs/amazon-kinesis-client/issues/528)
|
||||
|
||||
### Release 2.1.3 (March 18, 2019)
|
||||
[Milestone#30](https://github.com/awslabs/amazon-kinesis-client/milestone/30)
|
||||
* Added a message to recommend using `KinesisClientUtil` when an acquire timeout occurs in the `FanOutRecordsPublisher`.
|
||||
* [PR#514](https://github.com/awslabs/amazon-kinesis-client/pull/514)
|
||||
* Added a sleep between retries while waiting for a newly created stream consumer to become active.
|
||||
* [PR#506](https://github.com/awslabs/amazon-kinesis-client/issues/506)
|
||||
* Added timeouts on all futures returned from the DynamoDB and Kinesis clients.
|
||||
The timeouts can be configured by setting `LeaseManagementConfig#requestTimeout(Duration)` for DynamoDB, and `PollingConfig#kinesisRequestTimeout(Duration)` for Kinesis.
|
||||
* [PR#518](https://github.com/awslabs/amazon-kinesis-client/pull/518)
|
||||
* Upgraded to SDK version 2.5.10.
|
||||
* [PR#518](https://github.com/awslabs/amazon-kinesis-client/pull/518)
|
||||
* Artifacts for the Amazon Kinesis Client for Java are now signed by a new GPG key:
|
||||
```
|
||||
pub 4096R/86368934 2019-02-14 [expires: 2020-02-14]
|
||||
uid Amazon Kinesis Tools <amazon-kinesis-tools@amazon.com>
|
||||
```
|
||||
|
||||
### Release 2.1.2 (February 18, 2019)
|
||||
[Milestone#29](https://github.com/awslabs/amazon-kinesis-client/milestone/29)
|
||||
* Fixed handling of the progress detection in the `ShardConsumer` to restart from the last accepted record, instead of the last queued record.
|
||||
* [PR#492](https://github.com/awslabs/amazon-kinesis-client/pull/492)
|
||||
* Fixed handling of exceptions when using polling so that it will no longer treat `SdkException`s as an unexpected exception.
|
||||
* [PR#497](https://github.com/awslabs/amazon-kinesis-client/pull/497)
|
||||
* [PR#502](https://github.com/awslabs/amazon-kinesis-client/pull/502)
|
||||
* Fixed a case where lease loss would block the `Scheduler` while waiting for a record processor's `processRecords` method to complete.
|
||||
* [PR#501](https://github.com/awslabs/amazon-kinesis-client/pull/501)
|
||||
|
||||
### Release 2.1.1 (February 6, 2019)
|
||||
[Milestone#28](https://github.com/awslabs/amazon-kinesis-client/milestone/28)
|
||||
* Introducing `SHUT_DOWN_STARTED` state for the `WorkerStateChangeListener`.
|
||||
* [PR#457](https://github.com/awslabs/amazon-kinesis-client/pull/457)
|
||||
* Fixed a bug with `AWSSessionCredentials` using `AWSSecretID` instead of `AWSAccessID` and vice versa.
|
||||
* [PR#486](https://github.com/awslabs/amazon-kinesis-client/pull/486)
|
||||
* Upgrading SDK version to 2.4.0, which includes a fix for a possible deadlock when using Enhanced Fan-Out.
|
||||
* [PR#493](https://github.com/awslabs/amazon-kinesis-client/pull/493)
|
||||
|
||||
### Release 2.1.0 (January 14, 2019)
|
||||
[Milestone #27](https://github.com/awslabs/amazon-kinesis-client/milestone/27)
|
||||
* Introducing MultiLangDaemon support for Enhanced Fan-Out.
|
||||
* MultiLangDaemon now supports the following command line options.
|
||||
* `--properties-file`: Properties file that the KCL should use to set up the Scheduler.
|
||||
* `--log-configuration`: logback.xml that the KCL should use for logging.
|
||||
* Updated AWS SDK dependency to 2.2.0.
|
||||
* MultiLangDaemon now uses logback for logging.
|
||||
|
||||
### Release 2.0.5 (November 12, 2018)
|
||||
[Milestone #26](https://github.com/awslabs/amazon-kinesis-client/milestone/26?closed=1)
|
||||
* Fixed a deadlock condition that could occur when using the polling model.
|
||||
When using the `PollingConfig` and a slower record processor it was possible to hit a deadlock in the retrieval of records.
|
||||
* [PR #462](https://github.com/awslabs/amazon-kinesis-client/pull/462)
|
||||
* [Issue #448](https://github.com/awslabs/amazon-kinesis-client/issues/448)
|
||||
* Adjusted `RetrievalConfig`, and `FanOutConfig` to use accessors instead of direct member access.
|
||||
* [PR #453](https://github.com/awslabs/amazon-kinesis-client/pull/453)
|
||||
|
||||
|
||||
### Release 2.0.4 (October 18, 2018)
|
||||
[Milestone #25](https://github.com/awslabs/amazon-kinesis-client/milestone/25)
|
||||
* Added method to retrieve leases from the LeaseCoordinator and LeaseTaker.
|
||||
* [PR #428](https://github.com/awslabs/amazon-kinesis-client/pull/428)
|
||||
* Fixed a race condition shutting down the Scheduler before it has completed initialization.
|
||||
* [PR #439](https://github.com/awslabs/amazon-kinesis-client/pull/439)
|
||||
* [Issue #427](https://github.com/awslabs/amazon-kinesis-client/issues/427)
|
||||
* Added `HierarchicalShardSyncer` which replaces the static `ShardSyncer`.
|
||||
`HierarchicalShardSyncer` removes the contention between multiple instances of the Scheduler when running under a single JVM.
|
||||
* [PR #395](https://github.com/awslabs/amazon-kinesis-client/pull/395)
|
||||
* [Issue #415](https://github.com/awslabs/amazon-kinesis-client/issues/415)
|
||||
* Added `TaskExecutionListener` which allows monitoring of tasks being executed by the `ShardConsumer`.
|
||||
The listener is invoked before and after a task is executed by the `ShardConsumer`.
|
||||
* [PR #417](https://github.com/awslabs/amazon-kinesis-client/pull/417)
|
||||
|
||||
### Release 2.0.3 (October 8, 2018)
|
||||
[Milestone #23](https://github.com/awslabs/amazon-kinesis-client/milestone/23)
|
||||
* Fixed an issue where the `KinesisAsyncClient` could be misconfigured to use HTTP 1.1.
|
||||
Using HTTP 1.1 with `SubscribeToShard` is unsupported, and could cause misdelivery of records to the record processor.
|
||||
* [Issue #391](https://github.com/awslabs/amazon-kinesis-client/issues/391)
|
||||
* [PR #434](https://github.com/awslabs/amazon-kinesis-client/pull/434)
|
||||
* [PR #433](https://github.com/awslabs/amazon-kinesis-client/pull/433)
|
||||
* Lower the severity of `ReadTimeout` exceptions.
|
||||
`ReadTimeout` exceptions can occur if the client is unable to request data from Kinesis for more than client timeout, which defaults to 30 seconds. This can occur if the record processor blocks for more than the timeout period. `ReadTimeout` could also occur as part of [Issue #391](https://github.com/awslabs/amazon-kinesis-client/issues/391).
|
||||
* [Issue #399](https://github.com/awslabs/amazon-kinesis-client/issues/399)
|
||||
* [PR #403](https://github.com/awslabs/amazon-kinesis-client/pull/403)
|
||||
* Added a callback that allows applications to take actions after DynamoDB table creation.
|
||||
Applications can now install a callback that is called after creating the DynamoDB table by implementing `TableCreatorCallback`.
|
||||
* [PR #413](https://github.com/awslabs/amazon-kinesis-client/pull/413)
|
||||
* Updated the guava dependency to 26.0-jre.
|
||||
* [PR #420](https://github.com/awslabs/amazon-kinesis-client/pull/420)
|
||||
* [Issue #416](https://github.com/awslabs/amazon-kinesis-client/issues/416)
|
||||
* Added some additional debug logging around the initialization of the `FanOutRecordsPublisher`.
|
||||
* [PR #398](https://github.com/awslabs/amazon-kinesis-client/pull/398)
|
||||
* Upgraded AWS SDK version to 2.0.6
|
||||
* [PR #434](https://github.com/awslabs/amazon-kinesis-client/pull/434)
|
||||
|
||||
|
||||
### Release 2.0.2 (September 4, 2018)
|
||||
[Milestone #22](https://github.com/awslabs/amazon-kinesis-client/milestone/22)
|
||||
* Fixed an issue where the a warning would be logged every second if `logWarningForTaskAfterMillis` was set.
|
||||
The logging for last time of data arrival now respects the value of `logWarningForTaskAfterMillis`.
|
||||
* [PR #383](https://github.com/awslabs/amazon-kinesis-client/pull/383)
|
||||
* [Issue #381](https://github.com/awslabs/amazon-kinesis-client/issues/381)
|
||||
* Moved creation of `WorkerStateChangedListener` and `GracefulShutdownCoordinator` to the `CoordinatorConfig`.
|
||||
Originally the `WorkerStateChangedListener` and `GracefulShutdownCoordinator` were created by methods on the `SchedulerCoordinatorFactory`, but they should have been configuration options.
|
||||
The original methods have been deprecated, and may be removed at a later date.
|
||||
* [PR #385](https://github.com/awslabs/amazon-kinesis-client/pull/385)
|
||||
* [PR #388](https://github.com/awslabs/amazon-kinesis-client/pull/388)
|
||||
* Removed dependency on Apache Commons Lang 2.6.
|
||||
The dependency on Apache Commons Lang 2.6 has removed, and all usages updated to use Apache Commons Lang 3.7.
|
||||
* [PR #386](https://github.com/awslabs/amazon-kinesis-client/pull/386)
|
||||
* [Issue #370](https://github.com/awslabs/amazon-kinesis-client/issues/370)
|
||||
* Fixed a typo in the MutliLang Daemon shutdown hook.
|
||||
* [PR #387](https://github.com/awslabs/amazon-kinesis-client/pull/387)
|
||||
* Added method `onAllInitializationAttemptsFailed(Throwable)` to `WorkerStateChangedListener` to report when all initialization attempts have failed.
|
||||
This method is a default method, and it isn't require to implement the method. This method is only called after all attempts to initialize the `Scheduler` have failed.
|
||||
* [PR #369](https://github.com/awslabs/amazon-kinesis-client/pull/369)
|
||||
|
||||
### Release 2.0.1 (August 21, 2018)
|
||||
* Mark certain internal components with `@KinesisClientInternalApi` attribute.
|
||||
Components marked as internal may be deprecated at a faster rate than public components.
|
||||
* [PR #358](https://github.com/awslabs/amazon-kinesis-client/pull/358)
|
||||
* Fixed an issue where `ResourceNotFoundException` on subscription to a shard was not triggering end of shard handling.
|
||||
If a lease table contains a shard that is no longer present in the stream attempt to subscribe to that shard will trigger a `ResourceNotFoundException`. These exception are treated the same as reaching the end of a shard.
|
||||
* [PR #359](https://github.com/awslabs/amazon-kinesis-client/pull/359)
|
||||
* Fixed an issue where the KCL would not Use the configured DynamoDB IOPs when creating the lease table.
|
||||
* [PR #360](https://github.com/awslabs/amazon-kinesis-client/pull/360)
|
||||
* Make the maximum number of Scheduler initialization attempts configurable.
|
||||
The maximum number of `Scheduler` initialization attempts can be configured via `CoordinatorConfig#maxInitializationAttempts`.
|
||||
* [PR #363](https://github.com/awslabs/amazon-kinesis-client/pull/363)
|
||||
* [PR #368](https://github.com/awslabs/amazon-kinesis-client/pull/368)
|
||||
* Fixed an issue where it was possible to get a duplicate record when resubscribing to a shard.
|
||||
Subscribe to shard requires periodic resubscribing, and uses a new concept of a continuation sequence number. If the continuation sequence number was equal to the last record that record would be processed a second time. Resubscribing now uses `AFTER_SEQUENCE_NUMBER` to ensure that only later records are returned.
|
||||
* [PR #371](https://github.com/awslabs/amazon-kinesis-client/pull/371)
|
||||
* Upgraded to AWS SDK 2.0.1
|
||||
* [PR #372](https://github.com/awslabs/amazon-kinesis-client/pull/372)
|
||||
* Fixed an issue where time based restart of the subscription wasn't resetting the `lastRequestTime`.
|
||||
If a subscription hasn't delivered any data for more than 30 seconds it will be canceled and restarted. This detection is based of the `lastRequestTime` which wasn't getting reset after the restart was triggered.
|
||||
* [PR #373](https://github.com/awslabs/amazon-kinesis-client/pull/373)
|
||||
* Fixed an issue where requesting on the subscription from the `FanOutRecordsPublisher` could trigger an unexpected failure.
|
||||
Due to a race condition the underlying flow in the subscription could be set to something else. The method is now synchronized, and verifies that the subscriber it was created with is still the subscriber in affect.
|
||||
This issue generally would only appear when multiple errors were occurring while connecting to Kinesis.
|
||||
* [PR #374](https://github.com/awslabs/amazon-kinesis-client/pull/374)
|
||||
* Fixed an issue where the number of requested items could exceed the capacity of the RxJava queue.
|
||||
There was an off by one issue when determining whether to make a request to the SDK subscription. This changes the calculation to represent the capacity as a queue.
|
||||
* [PR #375](https://github.com/awslabs/amazon-kinesis-client/pull/375)
|
||||
|
||||
### Release 2.0.0 (August 02, 2018)
|
||||
* The Maven `groupId`, along with the `version`, for the Amazon Kinesis Client has changed from `com.amazonaws` to `software.amazon.kinesis`.
|
||||
To add a dependency on the new version of the Amazon Kinesis Client:
|
||||
``` xml
|
||||
<dependency>
|
||||
<groupId>software.amazon.kinesis</groupId>
|
||||
<artifactId>amazon-kinesis-client</artifactId>
|
||||
<version>2.0.0</version>
|
||||
</dependency>
|
||||
```
|
||||
* Added support for Enhanced Fan Out.
|
||||
Enhanced Fan Out provides for lower end to end latency, and increased number of consumers per stream.
|
||||
* Records are now delivered via streaming, reducing end-to-end latency.
|
||||
* The Amazon Kinesis Client will automatically register a new consumer if required.
|
||||
When registering a new consumer, the Kinesis Client will default to the application name unless configured otherwise.
|
||||
* `SubscribeToShard` maintains long lived connections with Kinesis, which in the AWS Java SDK 2.0 is limited by default.
|
||||
The `KinesisClientUtil` has been added to assist configuring the `maxConcurrency` of the `KinesisAsyncClient`.
|
||||
__WARNING: The Amazon Kinesis Client may see significantly increased latency, unless the `KinesisAsyncClient` is configured to have a `maxConcurrency` high enough to allow all leases plus additional usages of the `KinesisAsyncClient`.__
|
||||
* The Amazon Kinesis Client now uses 3 additional Kinesis API's:
|
||||
__WARNING: If using a restrictive Kinesis IAM policy you may need to add the following API methods to the policy.__
|
||||
* [`SubscribeToShard`](https://docs.aws.amazon.com/kinesis/latest/APIReference/API_SubscribeToShard.html)
|
||||
* [`DescribeStreamSummary`](https://docs.aws.amazon.com/kinesis/latest/APIReference/API_DescribeStreamSummary.html)
|
||||
* [`DescribeStreamConsumer`](https://docs.aws.amazon.com/kinesis/latest/APIReference/API_DescribeStreamConsumer.html)
|
||||
* [`RegisterStreamConsumer`](https://docs.aws.amazon.com/kinesis/latest/APIReference/API_RegisterStreamConsumer.html)
|
||||
* New configuration options are available to configure Enhanced Fan Out.
|
||||
|
||||
| Name | Default | Description |
|
||||
|-----------------|---------|---------------------------------------------------------------------------------------------------------------------|
|
||||
| consumerArn | Unset | The ARN for an already created consumer. If this is set, the Kinesis Client will not attempt to create a consumer. |
|
||||
| streamName | Unset | The name of the stream that a consumer should be create for if necessary |
|
||||
| consumerName | Unset | The name of the consumer to create. If this is not set the applicationName will be used instead. |
|
||||
| applicationName | Unset | The name of the application. This is used as the name of the consumer unless consumerName is set. |
|
||||
|
||||
* Modular Configuration of the Kinesis Client
|
||||
The Kinesis Client has migrated to a modular configuration system, and the `KinesisClientLibConfiguration` class has been removed.
|
||||
Configuration has been split into 7 classes. Default versions of the configuration can be created from the `ConfigsBuilder`.
|
||||
Please [see the migration guide for more information][migration-guide].
|
||||
* `CheckpointConfig`
|
||||
* `CoordinatorConfig`
|
||||
* `LeaseManagementConfig`
|
||||
* `LifecycleConfig`
|
||||
* `MetricsConfig`
|
||||
* `ProcessorConfig`
|
||||
* `RetrievalConfig`
|
||||
|
||||
* Upgraded to AWS Java SDK 2.0
|
||||
The Kinesis Client now uses the AWS Java SDK 2.0. The dependency on AWS Java SDK 1.11 has been removed.
|
||||
All configurations will only accept 2.0 clients.
|
||||
* When configuring the `KinesisAsyncClient` the `KinesisClientUtil#createKinesisAsyncClient` can be used to configure the Kinesis Client
|
||||
* __If you need support for AWS Java SDK 1.11 you will need to add a direct dependency.__
|
||||
__When adding a dependency you must ensure that the 1.11 versions of Jackson dependencies are excluded__
|
||||
[Please see the migration guide for more information][migration-guide]
|
||||
|
||||
* MultiLangDaemon is now a separate module
|
||||
The MultiLangDaemon has been separated to its own Maven module and is no longer available in `amazon-kinesis-client`. To include the MultiLangDaemon, add a dependency on `amazon-kinesis-client-multilang`.
|
||||
|
||||
[kinesis]: http://aws.amazon.com/kinesis
|
||||
[kinesis-forum]: http://developer.amazonwebservices.com/connect/forum.jspa?forumID=169
|
||||
[kinesis-client-library-issues]: https://github.com/awslabs/amazon-kinesis-client/issues
|
||||
[docs-signup]: http://docs.aws.amazon.com/AWSSdkDocsJava/latest/DeveloperGuide/java-dg-setup.html
|
||||
[kinesis-guide]: http://docs.aws.amazon.com/kinesis/latest/dev/introduction.html
|
||||
[kinesis-guide-begin]: http://docs.aws.amazon.com/kinesis/latest/dev/before-you-begin.html
|
||||
[kinesis-guide-create]: http://docs.aws.amazon.com/kinesis/latest/dev/step-one-create-stream.html
|
||||
[kinesis-guide-applications]: http://docs.aws.amazon.com/kinesis/latest/dev/kinesis-record-processor-app.html
|
||||
[kinesis-guide-monitoring-with-kcl]: http://docs.aws.amazon.com//kinesis/latest/dev/monitoring-with-kcl.html
|
||||
[kinesis-guide-kpl]: http://docs.aws.amazon.com//kinesis/latest/dev/developing-producers-with-kpl.html
|
||||
[kinesis-guide-consumer-deaggregation]: http://docs.aws.amazon.com//kinesis/latest/dev/kinesis-kpl-consumer-deaggregation.html
|
||||
[kclpy]: https://github.com/awslabs/amazon-kinesis-client-python
|
||||
[multi-lang-protocol]: https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/multilang/package-info.java
|
||||
### Release 3.0.3 (May 7, 2025)
|
||||
* [#1464](https://github.com/awslabs/amazon-kinesis-client/pull/1464) Add config for LeaseAssignmentManager frequency and improve assignment time of newly created leases
|
||||
* [#1463](https://github.com/awslabs/amazon-kinesis-client/pull/1463) Extend ShardConsumer constructor to have ConsumerTaskFactory as a parameter to support [DynamoDB Streams Kinesis Adapter](https://github.com/awslabs/dynamodb-streams-kinesis-adapter) compatibility
|
||||
* [#1472](https://github.com/awslabs/amazon-kinesis-client/pull/1472) Remove unused synchronized keyword
|
||||
|
||||
### Release 3.0.2 (March 12, 2025)
|
||||
* [#1443](https://github.com/awslabs/amazon-kinesis-client/pull/1443) Reduce DynamoDB IOPS for smaller stream and worker count applications
|
||||
* The below two PRs are intended to support [DynamoDB Streams Kinesis Adapter](https://github.com/awslabs/dynamodb-streams-kinesis-adapter) compatibility
|
||||
* [#1441](https://github.com/awslabs/amazon-kinesis-client/pull/1441) Make consumerTaskFactory overridable by customers
|
||||
* [#1440](https://github.com/awslabs/amazon-kinesis-client/pull/1440) Make ShutdownTask, ProcessTask, InitializeTask, BlockOnParentTask, and ShutdownNotificationTask overridable by customers
|
||||
* [#1437](https://github.com/awslabs/amazon-kinesis-client/pull/1437) Suppress LeaseAssignmentManager excessive WARN logs when there is no issue
|
||||
* [#1439](https://github.com/awslabs/amazon-kinesis-client/pull/1439) Upgrade io.netty:netty-handler from 4.1.108.Final to 4.1.118.Final
|
||||
* [#1400](https://github.com/awslabs/amazon-kinesis-client/pull/1400) Upgrade com.fasterxml.jackson.core:jackson-databind from 2.10.1 to 2.12.7.1
|
||||
|
||||
### Release 3.0.1 (November 14, 2024)
|
||||
* [#1401](https://github.com/awslabs/amazon-kinesis-client/pull/1401) Fixed the lease graceful handoff behavior in the multi-stream processing mode
|
||||
* [#1398](https://github.com/awslabs/amazon-kinesis-client/pull/1398) Addressed several KCL 3.0 related issues raised via GitHub
|
||||
* Fixed transitive dependencies and added a Maven plugin to catch potential transitive dependency issues at build time
|
||||
* Removed the redundant shutdown of the leaseCoordinatorThreadPool
|
||||
* Fixed typo THROUGHOUT_PUT_KBPS
|
||||
* Fixed issues in scheduler shutdown sequence
|
||||
|
||||
* Note: If you are using [multi-stream processing with KCL](https://docs.aws.amazon.com/streams/latest/dev/kcl-multi-stream.html), you need to use the release 3.0.1 or later.
|
||||
|
||||
### Release 3.0.0 (November 06, 2024)
|
||||
* New lease assignment / load balancing algorithm
|
||||
* KCL 3.x introduces a new lease assignment and load balancing algorithm. It assigns leases among workers based on worker utilization metrics and throughput on each lease, replacing the previous lease count-based lease assignment algorithm.
|
||||
* When KCL detects higher variance in CPU utilization among workers, it proactively reassigns leases from over-utilized workers to under-utilized workers for even load balancing. This ensures even CPU utilization across workers and removes the need to over-provision the stream processing compute hosts.
|
||||
* Optimized DynamoDB RCU usage
|
||||
* KCL 3.x optimizes DynamoDB read capacity unit (RCU) usage on the lease table by implementing a global secondary index with leaseOwner as the partition key. This index mirrors the leaseKey attribute from the base lease table, allowing workers to efficiently discover their assigned leases by querying the index instead of scanning the entire table.
|
||||
* This approach significantly reduces read operations compared to earlier KCL versions, where workers performed full table scans, resulting in higher RCU consumption.
|
||||
* Graceful lease handoff
|
||||
* KCL 3.x introduces a feature called "graceful lease handoff" to minimize data reprocessing during lease reassignments. Graceful lease handoff allows the current worker to complete checkpointing of processed records before transferring the lease to another worker. For graceful lease handoff, you should implement checkpointing logic within the existing `shutdownRequested()` method.
|
||||
* This feature is enabled by default in KCL 3.x, but you can turn off this feature by adjusting the configuration property `isGracefulLeaseHandoffEnabled`.
|
||||
* While this approach significantly reduces the probability of data reprocessing during lease transfers, it doesn't completely eliminate the possibility. To maintain data integrity and consistency, it's crucial to design your downstream consumer applications to be idempotent. This ensures that the application can handle potential duplicate record processing without adverse effects.
|
||||
* New DynamoDB metadata management artifacts
|
||||
* KCL 3.x introduces two new DynamoDB tables for improved lease management:
|
||||
* Worker metrics table: Records CPU utilization metrics from each worker. KCL uses these metrics for optimal lease assignments, balancing resource utilization across workers. If CPU utilization metric is not available, KCL assigns leases to balance the total sum of shard throughput per worker instead.
|
||||
* Coordinator state table: Stores internal state information for workers. Used to coordinate in-place migration from KCL 2.x to KCL 3.x and leader election among workers.
|
||||
* Follow this [documentation](https://docs.aws.amazon.com/streams/latest/dev/kcl-migration-from-2-3.html#kcl-migration-from-2-3-IAM-permissions) to add required IAM permissions for your KCL application.
|
||||
* Other improvements and changes
|
||||
* Dependency on the AWS SDK for Java 1.x has been fully removed.
|
||||
* The Glue Schema Registry integration functionality no longer depends on AWS SDK for Java 1.x. Previously, it required this as a transient dependency.
|
||||
* Multilangdaemon has been upgraded to use AWS SDK for Java 2.x. It no longer depends on AWS SDK for Java 1.x.
|
||||
* `idleTimeBetweenReadsInMillis` (PollingConfig) now has a minimum default value of 200.
|
||||
* This polling configuration property determines the [publishers](https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/retrieval/polling/PrefetchRecordsPublisher.java) wait time between GetRecords calls in both success and failure cases. Previously, setting this value below 200 caused unnecessary throttling. This is because Amazon Kinesis Data Streams supports up to five read transactions per second per shard for shared-throughput consumers.
|
||||
* Shard lifecycle management is improved to deal with edge cases around shard splits and merges to ensure records continue being processed as expected.
|
||||
* Migration
|
||||
* The programming interfaces of KCL 3.x remain identical with KCL 2.x for an easier migration, with the exception of those applications that do not use the recommended approach of using the Config Builder. These applications will have to refer to [the troubleshooting guide](https://docs.aws.amazon.com/streams/latest/dev/troubleshooting-consumers.html#compiliation-error-leasemanagementconfig). For detailed migration instructions, please refer to the [Migrate consumers from KCL 2.x to KCL 3.x](https://docs.aws.amazon.com/streams/latest/dev/kcl-migration-from-2-3.html) page in the Amazon Kinesis Data Streams developer guide.
|
||||
* Configuration properties
|
||||
* New configuration properties introduced in KCL 3.x are listed in this [doc](https://github.com/awslabs/amazon-kinesis-client/blob/master/docs/kcl-configurations.md#new-configurations-in-kcl-3x).
|
||||
* Deprecated configuration properties in KCL 3.x are listed in this [doc](https://github.com/awslabs/amazon-kinesis-client/blob/master/docs/kcl-configurations.md#discontinued-configuration-properties-in-kcl-3x). You need to keep the deprecated configuration properties during the migration from any previous KCL version to KCL 3.x.
|
||||
* Metrics
|
||||
* New CloudWatch metrics introduced in KCL 3.x are explained in the [Monitor the Kinesis Client Library with Amazon CloudWatch](https://docs.aws.amazon.com/streams/latest/dev/monitoring-with-kcl.html) in the Amazon Kinesis Data Streams developer guide. The following operations are newly added in KCL 3.x:
|
||||
* `LeaseAssignmentManager`
|
||||
* `WorkerMetricStatsReporter`
|
||||
* `LeaseDiscovery`
|
||||
|
|
|
|||
116
README.md
116
README.md
|
|
@ -1,46 +1,59 @@
|
|||
# Amazon Kinesis Client Library for Java
|
||||
[](https://travis-ci.org/awslabs/amazon-kinesis-client)
|
||||
|
||||
The **Amazon Kinesis Client Library for Java** (Amazon KCL) enables Java developers to easily consume and process data from [Amazon Kinesis][kinesis].
|
||||
> [!IMPORTANT]
|
||||
> ### Amazon Kinesis Client Library (KCL) 1.x will reach end-of-support on January 30, 2026
|
||||
> Amazon Kinesis Client Library (KCL) 1.x will reach end-of-support on January 30, 2026. Accordingly, these versions will enter maintenance mode on April 17, 2025. During maintenance mode, AWS will provide updates only for critical bug fixes and security issues. Major versions in maintenance mode will not receive updates for new features or feature enhancements. If you’re using KCL 1.x, we recommend migrating to the latest versions. When migrating from KCL 1.x to 3.x, you will need to update interfaces and security credential providers in your application. For details about the end-of-support notice and required actions, see the following links:
|
||||
> * [AWS Blog: Announcing end-of-support for Amazon Kinesis Client Library 1.x and Amazon Kinesis Producer Library 0.x effective January 30, 2026](https://aws.amazon.com/blogs/big-data/announcing-end-of-support-for-amazon-kinesis-client-library-1-x-and-amazon-kinesis-producer-library-0-x-effective-january-30-2026/)
|
||||
> * [Kinesis documentation: KCL version lifecycle policy](https://docs.aws.amazon.com/streams/latest/dev/kcl-version-lifecycle-policy.html)
|
||||
> * [Kinesis documentation: Migrating from KCL 1.x to KCL 3.x](https://docs.aws.amazon.com/streams/latest/dev/kcl-migration-1-3.html)
|
||||
|
||||
* [Kinesis Product Page][kinesis]
|
||||
* [Forum][kinesis-forum]
|
||||
## Introduction
|
||||
|
||||
The **Amazon Kinesis Client Library (KCL) for Java** enables Java developers to easily consume and process data from [Amazon Kinesis Data Streams][kinesis].
|
||||
|
||||
* [Kinesis Data Streams Product Page][kinesis]
|
||||
* [Amazon re:Post Forum: Kinesis][kinesis-forum]
|
||||
* [Javadoc][kcl-javadoc]
|
||||
* [FAQ](docs/FAQ.md)
|
||||
* [Developer Guide - Kinesis Client Library][kcl-aws-doc]
|
||||
* [KCL GitHub documentation](docs/) (folder)
|
||||
* [Issues][kinesis-client-library-issues]
|
||||
|
||||
### Recommended Upgrade for All Users of the 1.x Amazon Kinesis Client
|
||||
:warning: We recommend customers to migrate to 1.14.1 or newer to avoid [known bugs](https://github.com/awslabs/amazon-kinesis-client/issues/778) in 1.14.0 version
|
||||
|
||||
### Recommended Upgrade for All Users of the 2.x Amazon Kinesis Client
|
||||
**:warning: It's highly recommended for users of version 2.0 of the Amazon Kinesis Client to upgrade to version 2.0.3 or later. A [bug has been](https://github.com/awslabs/amazon-kinesis-client/issues/391) identified in versions prior to 2.0.3 that could cause records to be delivered to the wrong record processor.**
|
||||
|
||||
**:information_source: Amazon Kinesis Client versions 1.x are not impacted.**
|
||||
|
||||
Please open an issue if you have any questions.
|
||||
* [Giving Feedback][giving-feedback]
|
||||
|
||||
## Features
|
||||
|
||||
* Provides an easy-to-use programming model for processing data using Amazon Kinesis
|
||||
* Helps with scale-out and fault-tolerant processing
|
||||
* **Scalability:** KCL enables applications to scale dynamically by distributing the processing load across multiple workers. You can scale your application in or out, manually or with auto-scaling, without worrying about load redistribution.
|
||||
* **Load balancing:** KCL automatically balances the processing load across available workers, resulting in an even distribution of work across workers.
|
||||
* **Checkpointing:** KCL manages checkpointing of processed records, enabling applications to resume processing from their last sucessfully processed position.
|
||||
* **Fault tolerance:** KCL provides built-in fault tolerance mechanisms, making sure that data processing continues even if individual workers fail. KCL also provides at-least-once delivery.
|
||||
* **Handling stream-level changes:** KCL adapts to shard splits and merges that might occur due to changes in data volume. It maintains ordering by making sure that child shards are processed only after their parent shard is completed and checkpointed.
|
||||
* **Monitoring:** KCL integrates with Amazon CloudWatch for consumer-level monitoring.
|
||||
* **Multi-language support:** KCL natively supports Java and enables multiple non-Java programming languages through MultiLangDaemon.
|
||||
|
||||
## Getting Started
|
||||
|
||||
1. **Sign up for AWS** — Before you begin, you need an AWS account. For more information about creating an AWS account and retrieving your AWS credentials, see [AWS Account and Credentials][docs-signup] in the AWS SDK for Java Developer Guide.
|
||||
1. **Sign up for Amazon Kinesis** — Go to the Amazon Kinesis console to sign up for the service and create an Amazon Kinesis stream. For more information, see [Create an Amazon Kinesis Stream][kinesis-guide-create] in the Amazon Kinesis Developer Guide.
|
||||
1. **Minimum requirements** — To use the Amazon Kinesis Client Library, you'll need **Java 1.8+**. For more information about Amazon Kinesis Client Library requirements, see [Before You Begin][kinesis-guide-begin] in the Amazon Kinesis Developer Guide.
|
||||
1. **Using the Amazon Kinesis Client Library** — The best way to get familiar with the Amazon Kinesis Client Library is to read [Developing Record Consumer Applications][kinesis-guide-applications] in the Amazon Kinesis Developer Guide.
|
||||
2. **Sign up for Amazon Kinesis** — Go to the Amazon Kinesis console to sign up for the service and create an Amazon Kinesis stream. For more information, see [Create an Amazon Kinesis Stream][kinesis-guide-create] in the Amazon Kinesis Developer Guide.
|
||||
3. **Minimum requirements** — To use the Amazon Kinesis Client Library, you will need **Java 1.8+**. For more information about Amazon Kinesis Client Library requirements, see [Before You Begin][kinesis-guide-begin] in the Amazon Kinesis Developer Guide.
|
||||
4. **Using the Amazon Kinesis Client Library** — The best way to get familiar with the Amazon Kinesis Client Library is to read [Use Kinesis Client Library][kinesis-guide-applications] in the Amazon Kinesis Data Streams Developer Guide. For more information on core KCL concepts, please refer to the [KCL Concepts][kinesis-client-library-concepts] page.
|
||||
|
||||
## Building from Source
|
||||
|
||||
After you've downloaded the code from GitHub, you can build it using Maven. To disable GPG signing in the build, use
|
||||
this command: `mvn clean install -Dgpg.skip=true`. Note: This command runs Integration tests, which in turn creates AWS
|
||||
resources (which requires manual cleanup). Integration tests require valid AWS credentials need to be discovered at
|
||||
runtime. To skip running integration tests, add ` -DskipITs` option to the build command.
|
||||
After you have downloaded the code from GitHub, you can build it using Maven. To disable GPG signing in the build, use
|
||||
this command: `mvn clean install -Dgpg.skip=true`.
|
||||
Note: This command does not run integration tests.
|
||||
|
||||
To disable running unit tests in the build, add the property `-Dskip.ut=true`.
|
||||
|
||||
## Running Integration Tests
|
||||
|
||||
To run integration tests: `mvn -Dit.test=*IntegrationTest verify`.
|
||||
Note that running integration tests creates AWS resources.
|
||||
Integration tests require valid AWS credentials.
|
||||
This will look for a default AWS profile specified in your local `.aws/credentials`.
|
||||
Optionally, you can provide the name of an IAM user/role to run tests with as a string using this command: `mvn -Dit.test=*IntegrationTest -DawsProfile="<PROFILE_NAME>" verify`.
|
||||
To run all integration tests: `mvn verify -DskipITs=false`.
|
||||
To run one integration tests, specify the integration test class: `mvn -Dit.test="BasicStreamConsumerIntegrationTest" -DskipITs=false verify`
|
||||
Optionally, you can provide the name of an IAM user/role to run tests with as a string using this command: `mvn -DskipITs=false -DawsProfile="<PROFILE_NAME>" verify`.
|
||||
|
||||
## Integration with the Kinesis Producer Library
|
||||
For producer-side developers using the **[Kinesis Producer Library (KPL)][kinesis-guide-kpl]**, the KCL integrates without additional effort. When the KCL retrieves an aggregated Amazon Kinesis record consisting of multiple KPL user records, it will automatically invoke the KPL to extract the individual user records before returning them to the user.
|
||||
|
|
@ -51,15 +64,32 @@ To make it easier for developers to write record processors in other languages,
|
|||
## Using the KCL
|
||||
The recommended way to use the KCL for Java is to consume it from Maven.
|
||||
|
||||
### Version 2.x
|
||||
## KCL versions
|
||||
|
||||
> [!WARNING]
|
||||
> ### Do not use AWS SDK for Java versions 2.27.19 to 2.27.23 with KCL 3.x
|
||||
> When using KCL 3.x with AWS SDK for Java versions 2.27.19 through 2.27.23, you may encounter the following DynamoDB exception:
|
||||
> ```software.amazon.awssdk.services.dynamodb.model.DynamoDbException: The document path provided in the update expression is invalid for update (Service: DynamoDb, Status Code: 400, Request ID: xxx)```.
|
||||
> This error occurs due to [a known issue](https://github.com/aws/aws-sdk-java-v2/issues/5584) in the AWS SDK for Java that affects the DynamoDB metadata table managed by KCL 3.x. The issue was introduced in version 2.27.19 and impacts all versions up to 2.27.23. The issue has been resolved in the AWS SDK for Java version 2.27.24. For optimal performance and stability, we recommend upgrading to version 2.28.0 or later.
|
||||
|
||||
### Version 3.x
|
||||
``` xml
|
||||
<dependency>
|
||||
<groupId>software.amazon.kinesis</groupId>
|
||||
<artifactId>amazon-kinesis-client</artifactId>
|
||||
<version>2.5.1</version>
|
||||
<version>3.0.3</version>
|
||||
</dependency>
|
||||
```
|
||||
|
||||
### Version 2.x
|
||||
[Version 2.x tracking branch](https://github.com/awslabs/amazon-kinesis-client/tree/v2.x)
|
||||
``` xml
|
||||
<dependency>
|
||||
<groupId>software.amazon.kinesis</groupId>
|
||||
<artifactId>amazon-kinesis-client</artifactId>
|
||||
<version>2.7.0</version>
|
||||
</dependency>
|
||||
```
|
||||
### Version 1.x
|
||||
[Version 1.x tracking branch](https://github.com/awslabs/amazon-kinesis-client/tree/v1.x)
|
||||
``` xml
|
||||
|
|
@ -69,18 +99,33 @@ The recommended way to use the KCL for Java is to consume it from Maven.
|
|||
<version>1.14.1</version>
|
||||
</dependency>
|
||||
```
|
||||
|
||||
## Release Notes
|
||||
### Release Notes
|
||||
|
||||
| KCL Version | Changelog |
|
||||
| --- | --- |
|
||||
| 2.x | [master/CHANGELOG.md](CHANGELOG.md) |
|
||||
| 3.x | [master/CHANGELOG.md](CHANGELOG.md) |
|
||||
| 2.x | [v2.x/CHANGELOG.md](https://github.com/awslabs/amazon-kinesis-client/blob/v2.x/CHANGELOG.md) |
|
||||
| 1.x | [v1.x/CHANGELOG.md](https://github.com/awslabs/amazon-kinesis-client/blob/v1.x/CHANGELOG.md) |
|
||||
|
||||
[kinesis]: http://aws.amazon.com/kinesis
|
||||
[kinesis-forum]: http://developer.amazonwebservices.com/connect/forum.jspa?forumID=169
|
||||
[kinesis-client-library-issues]: https://github.com/awslabs/amazon-kinesis-client/issues
|
||||
### Version recommendation
|
||||
We recommend all users to migrate to the latest respective versions to avoid known issues and benefit from all improvements.
|
||||
|
||||
## Giving Feedback
|
||||
|
||||
Help Us Improve the Kinesis Client Library! Your involvement is crucial to enhancing the Kinesis Client Library. We invite you to join our community and contribute in the following ways:
|
||||
|
||||
* [Issue](https://github.com/awslabs/amazon-kinesis-client/issues) Reporting: This is our preferred method of communication. Use this channel to report bugs, suggest improvements, or ask questions.
|
||||
* Feature Requests: Share your ideas for new features or vote for existing proposals on our [Issues](https://github.com/awslabs/amazon-kinesis-client/issues) page. This helps us prioritize development efforts.
|
||||
* Participate in Discussions: Engage with other users and our team in our discussion forums.
|
||||
* Submit [Pull Requests](https://github.com/awslabs/amazon-kinesis-client/pulls): If you have developed a fix or improvement, we welcome your code contributions.
|
||||
|
||||
By participating through these channels, you play a vital role in shaping the future of the Kinesis Client Library. We value your input and look forward to collaborating with you!
|
||||
|
||||
[docs-signup]: http://docs.aws.amazon.com/AWSSdkDocsJava/latest/DeveloperGuide/java-dg-setup.html
|
||||
[kcl-javadoc]: https://javadoc.io/doc/software.amazon.kinesis/amazon-kinesis-client/
|
||||
[kinesis]: http://aws.amazon.com/kinesis
|
||||
[kinesis-client-library-issues]: https://github.com/awslabs/amazon-kinesis-client/issues
|
||||
[kinesis-forum]: http://developer.amazonwebservices.com/connect/forum.jspa?forumID=169
|
||||
[kinesis-guide]: http://docs.aws.amazon.com/kinesis/latest/dev/introduction.html
|
||||
[kinesis-guide-begin]: http://docs.aws.amazon.com/kinesis/latest/dev/before-you-begin.html
|
||||
[kinesis-guide-create]: http://docs.aws.amazon.com/kinesis/latest/dev/step-one-create-stream.html
|
||||
|
|
@ -89,5 +134,8 @@ The recommended way to use the KCL for Java is to consume it from Maven.
|
|||
[kinesis-guide-kpl]: http://docs.aws.amazon.com//kinesis/latest/dev/developing-producers-with-kpl.html
|
||||
[kinesis-guide-consumer-deaggregation]: http://docs.aws.amazon.com//kinesis/latest/dev/kinesis-kpl-consumer-deaggregation.html
|
||||
[kclpy]: https://github.com/awslabs/amazon-kinesis-client-python
|
||||
[multi-lang-protocol]: https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/package-info.java
|
||||
[migration-guide]: https://docs.aws.amazon.com/streams/latest/dev/kcl-migration.html
|
||||
[multi-lang-protocol]: /amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/package-info.java
|
||||
[migration-guide]: https://docs.aws.amazon.com/streams/latest/dev/kcl-migration-from-previous-versions
|
||||
[kcl-sample]: https://docs.aws.amazon.com/streams/latest/dev/kcl-example-code
|
||||
[kcl-aws-doc]: https://docs.aws.amazon.com/streams/latest/dev/kcl.html
|
||||
[giving-feedback]: https://github.com/awslabs/amazon-kinesis-client?tab=readme-ov-file#giving-feedback
|
||||
|
|
|
|||
|
|
@ -21,16 +21,12 @@
|
|||
<parent>
|
||||
<artifactId>amazon-kinesis-client-pom</artifactId>
|
||||
<groupId>software.amazon.kinesis</groupId>
|
||||
<version>2.5.2-SNAPSHOT</version>
|
||||
<version>3.0.3</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<artifactId>amazon-kinesis-client-multilang</artifactId>
|
||||
|
||||
<properties>
|
||||
<aws-java-sdk.version>1.12.405</aws-java-sdk.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>software.amazon.kinesis</groupId>
|
||||
|
|
@ -43,42 +39,16 @@
|
|||
<version>${awssdk.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.amazonaws</groupId>
|
||||
<artifactId>aws-java-sdk-core</artifactId>
|
||||
<version>${aws-java-sdk.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-databind</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>com.fasterxml.jackson.dataformat</groupId>
|
||||
<artifactId>jackson-dataformat-cbor</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.apache.httpcomponents</groupId>
|
||||
<artifactId>httpclient</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.amazonaws</groupId>
|
||||
<artifactId>aws-java-sdk-sts</artifactId>
|
||||
<version>${aws-java-sdk.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.projectlombok</groupId>
|
||||
<artifactId>lombok</artifactId>
|
||||
<version>1.18.24</version>
|
||||
<version>1.18.28</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>ch.qos.logback</groupId>
|
||||
<artifactId>logback-classic</artifactId>
|
||||
<version>1.3.0</version>
|
||||
<version>1.3.14</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.beust</groupId>
|
||||
|
|
@ -88,7 +58,7 @@
|
|||
<dependency>
|
||||
<groupId>commons-io</groupId>
|
||||
<artifactId>commons-io</artifactId>
|
||||
<version>2.11.0</version>
|
||||
<version>2.16.1</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
|
|
@ -104,6 +74,12 @@
|
|||
</dependency>
|
||||
|
||||
<!-- Test -->
|
||||
<dependency>
|
||||
<groupId>org.junit.jupiter</groupId>
|
||||
<artifactId>junit-jupiter-api</artifactId>
|
||||
<version>5.11.3</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
|
|
@ -122,6 +98,13 @@
|
|||
<version>1.3</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<!-- Using older version to be compatible with Java 8 -->
|
||||
<dependency>
|
||||
<groupId>org.mockito</groupId>
|
||||
<artifactId>mockito-junit-jupiter</artifactId>
|
||||
<version>3.12.4</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
|
|
@ -130,10 +113,9 @@
|
|||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.11.0</version>
|
||||
<version>3.13.0</version>
|
||||
<configuration>
|
||||
<source>1.8</source>
|
||||
<target>1.8</target>
|
||||
<release>8</release>
|
||||
<encoding>UTF-8</encoding>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
|
@ -143,7 +125,7 @@
|
|||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
<version>3.5.0</version>
|
||||
<version>3.7.0</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>attach-javadocs</id>
|
||||
|
|
@ -166,7 +148,29 @@
|
|||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>com.diffplug.spotless</groupId>
|
||||
<artifactId>spotless-maven-plugin</artifactId>
|
||||
<version>2.30.0</version> <!--last version to support java 8-->
|
||||
<configuration>
|
||||
<java>
|
||||
<palantirJavaFormat />
|
||||
<importOrder>
|
||||
<order>java,,\#</order>
|
||||
</importOrder>
|
||||
</java>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
<phase>compile</phase>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
|
||||
</build>
|
||||
|
||||
<profiles>
|
||||
|
|
|
|||
|
|
@ -18,11 +18,9 @@ import java.util.Date;
|
|||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.commons.lang3.Validate;
|
||||
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
|
||||
import lombok.Getter;
|
||||
import org.apache.commons.lang3.Validate;
|
||||
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
|
||||
import software.amazon.kinesis.checkpoint.ShardRecordProcessorCheckpointer;
|
||||
import software.amazon.kinesis.common.InitialPositionInStream;
|
||||
|
|
@ -119,14 +117,16 @@ public class KinesisClientLibConfiguration {
|
|||
/**
|
||||
* Metrics dimensions that always will be enabled regardless of the config provided by user.
|
||||
*/
|
||||
public static final Set<String> METRICS_ALWAYS_ENABLED_DIMENSIONS = ImmutableSet
|
||||
.of(MetricsUtil.OPERATION_DIMENSION_NAME);
|
||||
public static final Set<String> METRICS_ALWAYS_ENABLED_DIMENSIONS =
|
||||
ImmutableSet.of(MetricsUtil.OPERATION_DIMENSION_NAME);
|
||||
|
||||
/**
|
||||
* Allowed dimensions for CloudWatch metrics. By default, worker ID dimension will be disabled.
|
||||
*/
|
||||
public static final Set<String> DEFAULT_METRICS_ENABLED_DIMENSIONS = ImmutableSet.<String>builder()
|
||||
.addAll(METRICS_ALWAYS_ENABLED_DIMENSIONS).add(MetricsUtil.SHARD_ID_DIMENSION_NAME).build();
|
||||
.addAll(METRICS_ALWAYS_ENABLED_DIMENSIONS)
|
||||
.add(MetricsUtil.SHARD_ID_DIMENSION_NAME)
|
||||
.build();
|
||||
|
||||
/**
|
||||
* Metrics dimensions that signify all possible dimensions.
|
||||
|
|
@ -217,6 +217,9 @@ public class KinesisClientLibConfiguration {
|
|||
private AwsCredentialsProvider dynamoDBCredentialsProvider;
|
||||
private AwsCredentialsProvider cloudWatchCredentialsProvider;
|
||||
private long failoverTimeMillis;
|
||||
private boolean enablePriorityLeaseAssignment;
|
||||
private boolean leaseTableDeletionProtectionEnabled;
|
||||
private boolean leaseTablePitrEnabled;
|
||||
private String workerIdentifier;
|
||||
private long shardSyncIntervalMillis;
|
||||
private int maxRecords;
|
||||
|
|
@ -284,8 +287,8 @@ public class KinesisClientLibConfiguration {
|
|||
* @param workerId
|
||||
* Used to distinguish different workers/processes of a Kinesis application
|
||||
*/
|
||||
public KinesisClientLibConfiguration(String applicationName, String streamName,
|
||||
AwsCredentialsProvider credentialsProvider, String workerId) {
|
||||
public KinesisClientLibConfiguration(
|
||||
String applicationName, String streamName, AwsCredentialsProvider credentialsProvider, String workerId) {
|
||||
this(applicationName, streamName, credentialsProvider, credentialsProvider, credentialsProvider, workerId);
|
||||
}
|
||||
|
||||
|
|
@ -307,16 +310,36 @@ public class KinesisClientLibConfiguration {
|
|||
* @param workerId
|
||||
* Used to distinguish different workers/processes of a Kinesis application
|
||||
*/
|
||||
public KinesisClientLibConfiguration(String applicationName, String streamName,
|
||||
AwsCredentialsProvider kinesisCredentialsProvider, AwsCredentialsProvider dynamoDBCredentialsProvider,
|
||||
AwsCredentialsProvider cloudWatchCredentialsProvider, String workerId) {
|
||||
this(applicationName, streamName, null, null, DEFAULT_INITIAL_POSITION_IN_STREAM, kinesisCredentialsProvider,
|
||||
dynamoDBCredentialsProvider, cloudWatchCredentialsProvider, DEFAULT_FAILOVER_TIME_MILLIS, workerId,
|
||||
DEFAULT_MAX_RECORDS, DEFAULT_IDLETIME_BETWEEN_READS_MILLIS,
|
||||
DEFAULT_DONT_CALL_PROCESS_RECORDS_FOR_EMPTY_RECORD_LIST, DEFAULT_PARENT_SHARD_POLL_INTERVAL_MILLIS,
|
||||
DEFAULT_SHARD_SYNC_INTERVAL_MILLIS, DEFAULT_CLEANUP_LEASES_UPON_SHARDS_COMPLETION,
|
||||
DEFAULT_TASK_BACKOFF_TIME_MILLIS, DEFAULT_METRICS_BUFFER_TIME_MILLIS, DEFAULT_METRICS_MAX_QUEUE_SIZE,
|
||||
DEFAULT_VALIDATE_SEQUENCE_NUMBER_BEFORE_CHECKPOINTING, null, DEFAULT_SHUTDOWN_GRACE_MILLIS,
|
||||
public KinesisClientLibConfiguration(
|
||||
String applicationName,
|
||||
String streamName,
|
||||
AwsCredentialsProvider kinesisCredentialsProvider,
|
||||
AwsCredentialsProvider dynamoDBCredentialsProvider,
|
||||
AwsCredentialsProvider cloudWatchCredentialsProvider,
|
||||
String workerId) {
|
||||
this(
|
||||
applicationName,
|
||||
streamName,
|
||||
null,
|
||||
null,
|
||||
DEFAULT_INITIAL_POSITION_IN_STREAM,
|
||||
kinesisCredentialsProvider,
|
||||
dynamoDBCredentialsProvider,
|
||||
cloudWatchCredentialsProvider,
|
||||
DEFAULT_FAILOVER_TIME_MILLIS,
|
||||
workerId,
|
||||
DEFAULT_MAX_RECORDS,
|
||||
DEFAULT_IDLETIME_BETWEEN_READS_MILLIS,
|
||||
DEFAULT_DONT_CALL_PROCESS_RECORDS_FOR_EMPTY_RECORD_LIST,
|
||||
DEFAULT_PARENT_SHARD_POLL_INTERVAL_MILLIS,
|
||||
DEFAULT_SHARD_SYNC_INTERVAL_MILLIS,
|
||||
DEFAULT_CLEANUP_LEASES_UPON_SHARDS_COMPLETION,
|
||||
DEFAULT_TASK_BACKOFF_TIME_MILLIS,
|
||||
DEFAULT_METRICS_BUFFER_TIME_MILLIS,
|
||||
DEFAULT_METRICS_MAX_QUEUE_SIZE,
|
||||
DEFAULT_VALIDATE_SEQUENCE_NUMBER_BEFORE_CHECKPOINTING,
|
||||
null,
|
||||
DEFAULT_SHUTDOWN_GRACE_MILLIS,
|
||||
DEFAULT_SCHEDULER_INITIALIZATION_BACKOFF_TIME_MILLIS);
|
||||
}
|
||||
|
||||
|
|
@ -376,20 +399,53 @@ public class KinesisClientLibConfiguration {
|
|||
*/
|
||||
// CHECKSTYLE:IGNORE HiddenFieldCheck FOR NEXT 26 LINES
|
||||
// CHECKSTYLE:IGNORE ParameterNumber FOR NEXT 26 LINES
|
||||
public KinesisClientLibConfiguration(String applicationName, String streamName, String kinesisEndpoint,
|
||||
InitialPositionInStream initialPositionInStream, AwsCredentialsProvider kinesisCredentialsProvider,
|
||||
AwsCredentialsProvider dynamoDBCredentialsProvider, AwsCredentialsProvider cloudWatchCredentialsProvider,
|
||||
long failoverTimeMillis, String workerId, int maxRecords, long idleTimeBetweenReadsInMillis,
|
||||
boolean callProcessRecordsEvenForEmptyRecordList, long parentShardPollIntervalMillis,
|
||||
long shardSyncIntervalMillis, boolean cleanupTerminatedShardsBeforeExpiry, long taskBackoffTimeMillis,
|
||||
long metricsBufferTimeMillis, int metricsMaxQueueSize, boolean validateSequenceNumberBeforeCheckpointing,
|
||||
String regionName, long shutdownGraceMillis, long schedulerInitializationBackoffTimeMillis) {
|
||||
this(applicationName, streamName, kinesisEndpoint, null, initialPositionInStream, kinesisCredentialsProvider,
|
||||
dynamoDBCredentialsProvider, cloudWatchCredentialsProvider, failoverTimeMillis, workerId, maxRecords,
|
||||
idleTimeBetweenReadsInMillis, callProcessRecordsEvenForEmptyRecordList, parentShardPollIntervalMillis,
|
||||
shardSyncIntervalMillis, cleanupTerminatedShardsBeforeExpiry, taskBackoffTimeMillis,
|
||||
metricsBufferTimeMillis, metricsMaxQueueSize, validateSequenceNumberBeforeCheckpointing, regionName,
|
||||
shutdownGraceMillis, schedulerInitializationBackoffTimeMillis);
|
||||
public KinesisClientLibConfiguration(
|
||||
String applicationName,
|
||||
String streamName,
|
||||
String kinesisEndpoint,
|
||||
InitialPositionInStream initialPositionInStream,
|
||||
AwsCredentialsProvider kinesisCredentialsProvider,
|
||||
AwsCredentialsProvider dynamoDBCredentialsProvider,
|
||||
AwsCredentialsProvider cloudWatchCredentialsProvider,
|
||||
long failoverTimeMillis,
|
||||
String workerId,
|
||||
int maxRecords,
|
||||
long idleTimeBetweenReadsInMillis,
|
||||
boolean callProcessRecordsEvenForEmptyRecordList,
|
||||
long parentShardPollIntervalMillis,
|
||||
long shardSyncIntervalMillis,
|
||||
boolean cleanupTerminatedShardsBeforeExpiry,
|
||||
long taskBackoffTimeMillis,
|
||||
long metricsBufferTimeMillis,
|
||||
int metricsMaxQueueSize,
|
||||
boolean validateSequenceNumberBeforeCheckpointing,
|
||||
String regionName,
|
||||
long shutdownGraceMillis,
|
||||
long schedulerInitializationBackoffTimeMillis) {
|
||||
this(
|
||||
applicationName,
|
||||
streamName,
|
||||
kinesisEndpoint,
|
||||
null,
|
||||
initialPositionInStream,
|
||||
kinesisCredentialsProvider,
|
||||
dynamoDBCredentialsProvider,
|
||||
cloudWatchCredentialsProvider,
|
||||
failoverTimeMillis,
|
||||
workerId,
|
||||
maxRecords,
|
||||
idleTimeBetweenReadsInMillis,
|
||||
callProcessRecordsEvenForEmptyRecordList,
|
||||
parentShardPollIntervalMillis,
|
||||
shardSyncIntervalMillis,
|
||||
cleanupTerminatedShardsBeforeExpiry,
|
||||
taskBackoffTimeMillis,
|
||||
metricsBufferTimeMillis,
|
||||
metricsMaxQueueSize,
|
||||
validateSequenceNumberBeforeCheckpointing,
|
||||
regionName,
|
||||
shutdownGraceMillis,
|
||||
schedulerInitializationBackoffTimeMillis);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -448,15 +504,30 @@ public class KinesisClientLibConfiguration {
|
|||
*/
|
||||
// CHECKSTYLE:IGNORE HiddenFieldCheck FOR NEXT 26 LINES
|
||||
// CHECKSTYLE:IGNORE ParameterNumber FOR NEXT 26 LINES
|
||||
public KinesisClientLibConfiguration(String applicationName, String streamName, String kinesisEndpoint,
|
||||
String dynamoDBEndpoint, InitialPositionInStream initialPositionInStream,
|
||||
AwsCredentialsProvider kinesisCredentialsProvider, AwsCredentialsProvider dynamoDBCredentialsProvider,
|
||||
AwsCredentialsProvider cloudWatchCredentialsProvider, long failoverTimeMillis, String workerId,
|
||||
int maxRecords, long idleTimeBetweenReadsInMillis, boolean callProcessRecordsEvenForEmptyRecordList,
|
||||
long parentShardPollIntervalMillis, long shardSyncIntervalMillis,
|
||||
boolean cleanupTerminatedShardsBeforeExpiry, long taskBackoffTimeMillis, long metricsBufferTimeMillis,
|
||||
int metricsMaxQueueSize, boolean validateSequenceNumberBeforeCheckpointing, String regionName,
|
||||
long shutdownGraceMillis, long schedulerInitializationBackoffTimeMillis) {
|
||||
public KinesisClientLibConfiguration(
|
||||
String applicationName,
|
||||
String streamName,
|
||||
String kinesisEndpoint,
|
||||
String dynamoDBEndpoint,
|
||||
InitialPositionInStream initialPositionInStream,
|
||||
AwsCredentialsProvider kinesisCredentialsProvider,
|
||||
AwsCredentialsProvider dynamoDBCredentialsProvider,
|
||||
AwsCredentialsProvider cloudWatchCredentialsProvider,
|
||||
long failoverTimeMillis,
|
||||
String workerId,
|
||||
int maxRecords,
|
||||
long idleTimeBetweenReadsInMillis,
|
||||
boolean callProcessRecordsEvenForEmptyRecordList,
|
||||
long parentShardPollIntervalMillis,
|
||||
long shardSyncIntervalMillis,
|
||||
boolean cleanupTerminatedShardsBeforeExpiry,
|
||||
long taskBackoffTimeMillis,
|
||||
long metricsBufferTimeMillis,
|
||||
int metricsMaxQueueSize,
|
||||
boolean validateSequenceNumberBeforeCheckpointing,
|
||||
String regionName,
|
||||
long shutdownGraceMillis,
|
||||
long schedulerInitializationBackoffTimeMillis) {
|
||||
// Check following values are greater than zero
|
||||
checkIsValuePositive("FailoverTimeMillis", failoverTimeMillis);
|
||||
checkIsValuePositive("IdleTimeBetweenReadsInMillis", idleTimeBetweenReadsInMillis);
|
||||
|
|
@ -494,8 +565,8 @@ public class KinesisClientLibConfiguration {
|
|||
this.maxLeasesToStealAtOneTime = DEFAULT_MAX_LEASES_TO_STEAL_AT_ONE_TIME;
|
||||
this.initialLeaseTableReadCapacity = DEFAULT_INITIAL_LEASE_TABLE_READ_CAPACITY;
|
||||
this.initialLeaseTableWriteCapacity = DEFAULT_INITIAL_LEASE_TABLE_WRITE_CAPACITY;
|
||||
this.initialPositionInStreamExtended = InitialPositionInStreamExtended
|
||||
.newInitialPosition(initialPositionInStream);
|
||||
this.initialPositionInStreamExtended =
|
||||
InitialPositionInStreamExtended.newInitialPosition(initialPositionInStream);
|
||||
this.skipShardSyncAtWorkerInitializationIfLeasesExist = DEFAULT_SKIP_SHARD_SYNC_AT_STARTUP_IF_LEASES_EXIST;
|
||||
this.shardPrioritization = DEFAULT_SHARD_PRIORITIZATION;
|
||||
this.recordsFetcherFactory = new SimpleRecordsFetcherFactory();
|
||||
|
|
@ -558,15 +629,30 @@ public class KinesisClientLibConfiguration {
|
|||
*/
|
||||
// CHECKSTYLE:IGNORE HiddenFieldCheck FOR NEXT 26 LINES
|
||||
// CHECKSTYLE:IGNORE ParameterNumber FOR NEXT 26 LINES
|
||||
public KinesisClientLibConfiguration(String applicationName, String streamName, String kinesisEndpoint,
|
||||
String dynamoDBEndpoint, InitialPositionInStream initialPositionInStream,
|
||||
AwsCredentialsProvider kinesisCredentialsProvider, AwsCredentialsProvider dynamoDBCredentialsProvider,
|
||||
AwsCredentialsProvider cloudWatchCredentialsProvider, long failoverTimeMillis, String workerId,
|
||||
int maxRecords, long idleTimeBetweenReadsInMillis, boolean callProcessRecordsEvenForEmptyRecordList,
|
||||
long parentShardPollIntervalMillis, long shardSyncIntervalMillis,
|
||||
boolean cleanupTerminatedShardsBeforeExpiry, long taskBackoffTimeMillis, long metricsBufferTimeMillis,
|
||||
int metricsMaxQueueSize, boolean validateSequenceNumberBeforeCheckpointing, String regionName,
|
||||
RecordsFetcherFactory recordsFetcherFactory, long schedulerInitializationBackoffTimeMillis) {
|
||||
public KinesisClientLibConfiguration(
|
||||
String applicationName,
|
||||
String streamName,
|
||||
String kinesisEndpoint,
|
||||
String dynamoDBEndpoint,
|
||||
InitialPositionInStream initialPositionInStream,
|
||||
AwsCredentialsProvider kinesisCredentialsProvider,
|
||||
AwsCredentialsProvider dynamoDBCredentialsProvider,
|
||||
AwsCredentialsProvider cloudWatchCredentialsProvider,
|
||||
long failoverTimeMillis,
|
||||
String workerId,
|
||||
int maxRecords,
|
||||
long idleTimeBetweenReadsInMillis,
|
||||
boolean callProcessRecordsEvenForEmptyRecordList,
|
||||
long parentShardPollIntervalMillis,
|
||||
long shardSyncIntervalMillis,
|
||||
boolean cleanupTerminatedShardsBeforeExpiry,
|
||||
long taskBackoffTimeMillis,
|
||||
long metricsBufferTimeMillis,
|
||||
int metricsMaxQueueSize,
|
||||
boolean validateSequenceNumberBeforeCheckpointing,
|
||||
String regionName,
|
||||
RecordsFetcherFactory recordsFetcherFactory,
|
||||
long schedulerInitializationBackoffTimeMillis) {
|
||||
// Check following values are greater than zero
|
||||
checkIsValuePositive("FailoverTimeMillis", failoverTimeMillis);
|
||||
checkIsValuePositive("IdleTimeBetweenReadsInMillis", idleTimeBetweenReadsInMillis);
|
||||
|
|
@ -606,8 +692,8 @@ public class KinesisClientLibConfiguration {
|
|||
this.maxLeasesToStealAtOneTime = DEFAULT_MAX_LEASES_TO_STEAL_AT_ONE_TIME;
|
||||
this.initialLeaseTableReadCapacity = DEFAULT_INITIAL_LEASE_TABLE_READ_CAPACITY;
|
||||
this.initialLeaseTableWriteCapacity = DEFAULT_INITIAL_LEASE_TABLE_WRITE_CAPACITY;
|
||||
this.initialPositionInStreamExtended = InitialPositionInStreamExtended
|
||||
.newInitialPosition(initialPositionInStream);
|
||||
this.initialPositionInStreamExtended =
|
||||
InitialPositionInStreamExtended.newInitialPosition(initialPositionInStream);
|
||||
this.skipShardSyncAtWorkerInitializationIfLeasesExist = DEFAULT_SKIP_SHARD_SYNC_AT_STARTUP_IF_LEASES_EXIST;
|
||||
this.shardPrioritization = DEFAULT_SHARD_PRIORITIZATION;
|
||||
this.recordsFetcherFactory = recordsFetcherFactory;
|
||||
|
|
@ -932,8 +1018,8 @@ public class KinesisClientLibConfiguration {
|
|||
*/
|
||||
public KinesisClientLibConfiguration withInitialPositionInStream(InitialPositionInStream initialPositionInStream) {
|
||||
this.initialPositionInStream = initialPositionInStream;
|
||||
this.initialPositionInStreamExtended = InitialPositionInStreamExtended
|
||||
.newInitialPosition(initialPositionInStream);
|
||||
this.initialPositionInStreamExtended =
|
||||
InitialPositionInStreamExtended.newInitialPosition(initialPositionInStream);
|
||||
return this;
|
||||
}
|
||||
|
||||
|
|
@ -959,6 +1045,11 @@ public class KinesisClientLibConfiguration {
|
|||
return this;
|
||||
}
|
||||
|
||||
public KinesisClientLibConfiguration withEnablePriorityLeaseAssignment(boolean enablePriorityLeaseAssignment) {
|
||||
this.enablePriorityLeaseAssignment = enablePriorityLeaseAssignment;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param shardSyncIntervalMillis
|
||||
* Time between tasks to sync leases and Kinesis shards
|
||||
|
|
@ -977,6 +1068,10 @@ public class KinesisClientLibConfiguration {
|
|||
*/
|
||||
public KinesisClientLibConfiguration withMaxRecords(int maxRecords) {
|
||||
checkIsValuePositive("MaxRecords", (long) maxRecords);
|
||||
if (maxRecords > DEFAULT_MAX_RECORDS) {
|
||||
throw new IllegalArgumentException("maxRecords must be less than or equal to " + DEFAULT_MAX_RECORDS
|
||||
+ " but current value is " + maxRecords);
|
||||
}
|
||||
this.maxRecords = maxRecords;
|
||||
return this;
|
||||
}
|
||||
|
|
@ -1135,8 +1230,10 @@ public class KinesisClientLibConfiguration {
|
|||
} else if (metricsEnabledDimensions.contains(MetricsScope.METRICS_DIMENSIONS_ALL)) {
|
||||
this.metricsEnabledDimensions = METRICS_DIMENSIONS_ALL;
|
||||
} else {
|
||||
this.metricsEnabledDimensions = ImmutableSet.<String> builder().addAll(metricsEnabledDimensions)
|
||||
.addAll(METRICS_ALWAYS_ENABLED_DIMENSIONS).build();
|
||||
this.metricsEnabledDimensions = ImmutableSet.<String>builder()
|
||||
.addAll(metricsEnabledDimensions)
|
||||
.addAll(METRICS_ALWAYS_ENABLED_DIMENSIONS)
|
||||
.build();
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
|
@ -1267,7 +1364,8 @@ public class KinesisClientLibConfiguration {
|
|||
* @return this configuration object
|
||||
*/
|
||||
public KinesisClientLibConfiguration withMaxLeaseRenewalThreads(int maxLeaseRenewalThreads) {
|
||||
Validate.isTrue(maxLeaseRenewalThreads > 2,
|
||||
Validate.isTrue(
|
||||
maxLeaseRenewalThreads > 2,
|
||||
"The maximum number of lease renewal threads must be greater than or equal to 2.");
|
||||
this.maxLeaseRenewalThreads = maxLeaseRenewalThreads;
|
||||
|
||||
|
|
@ -1327,7 +1425,8 @@ public class KinesisClientLibConfiguration {
|
|||
* @return KinesisClientLibConfiguration
|
||||
*/
|
||||
public KinesisClientLibConfiguration withDataFetchingStrategy(String dataFetchingStrategy) {
|
||||
this.recordsFetcherFactory.dataFetchingStrategy(DataFetchingStrategy.valueOf(dataFetchingStrategy.toUpperCase()));
|
||||
this.recordsFetcherFactory.dataFetchingStrategy(
|
||||
DataFetchingStrategy.valueOf(dataFetchingStrategy.toUpperCase()));
|
||||
return this;
|
||||
}
|
||||
|
||||
|
|
@ -1413,7 +1512,8 @@ public class KinesisClientLibConfiguration {
|
|||
* Interval in milliseconds between retrying the scheduler initialization.
|
||||
* @return
|
||||
*/
|
||||
public KinesisClientLibConfiguration withSchedulerInitializationBackoffTimeMillis(long schedulerInitializationBackoffTimeMillis) {
|
||||
public KinesisClientLibConfiguration withSchedulerInitializationBackoffTimeMillis(
|
||||
long schedulerInitializationBackoffTimeMillis) {
|
||||
checkIsValuePositive("schedulerInitializationBackoffTimeMillis", schedulerInitializationBackoffTimeMillis);
|
||||
this.schedulerInitializationBackoffTimeMillis = schedulerInitializationBackoffTimeMillis;
|
||||
return this;
|
||||
|
|
|
|||
|
|
@ -23,8 +23,7 @@ import lombok.extern.slf4j.Slf4j;
|
|||
*/
|
||||
@Slf4j
|
||||
class DrainChildSTDERRTask extends LineReaderTask<Boolean> {
|
||||
DrainChildSTDERRTask() {
|
||||
}
|
||||
DrainChildSTDERRTask() {}
|
||||
|
||||
@Override
|
||||
protected HandleLineResult<Boolean> handleLine(String line) {
|
||||
|
|
|
|||
|
|
@ -37,8 +37,7 @@ import lombok.extern.slf4j.Slf4j;
|
|||
*/
|
||||
@Slf4j
|
||||
class DrainChildSTDOUTTask extends LineReaderTask<Boolean> {
|
||||
DrainChildSTDOUTTask() {
|
||||
}
|
||||
DrainChildSTDOUTTask() {}
|
||||
|
||||
@Override
|
||||
protected HandleLineResult<Boolean> handleLine(String line) {
|
||||
|
|
|
|||
|
|
@ -17,10 +17,9 @@ package software.amazon.kinesis.multilang;
|
|||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
|
||||
import software.amazon.kinesis.multilang.messages.Message;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import software.amazon.kinesis.multilang.messages.Message;
|
||||
|
||||
/**
|
||||
* Gets the next message off the STDOUT of the child process. Throws an exception if a message is not found before the
|
||||
|
|
@ -71,8 +70,10 @@ class GetNextMessageTask extends LineReaderTask<Message> {
|
|||
|
||||
@Override
|
||||
protected Message returnAfterException(Exception e) {
|
||||
throw new RuntimeException("Encountered an error while reading a line from STDIN for shard " + getShardId()
|
||||
+ " so won't be able to return a message.", e);
|
||||
throw new RuntimeException(
|
||||
"Encountered an error while reading a line from STDIN for shard " + getShardId()
|
||||
+ " so won't be able to return a message.",
|
||||
e);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -41,8 +41,7 @@ abstract class LineReaderTask<T> implements Callable<T> {
|
|||
|
||||
private String shardId;
|
||||
|
||||
LineReaderTask() {
|
||||
}
|
||||
LineReaderTask() {}
|
||||
|
||||
/**
|
||||
* Reads lines off the input stream until a return value is set, or an exception is encountered, or the end of the
|
||||
|
|
@ -180,5 +179,4 @@ abstract class LineReaderTask<T> implements Callable<T> {
|
|||
this.description = description;
|
||||
return this;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,8 +20,8 @@ import java.io.InputStreamReader;
|
|||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Future;
|
||||
|
||||
import software.amazon.kinesis.multilang.messages.Message;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import software.amazon.kinesis.multilang.messages.Message;
|
||||
|
||||
/**
|
||||
* Provides methods for interacting with the child process's STDOUT.
|
||||
|
|
@ -48,8 +48,7 @@ class MessageReader {
|
|||
/**
|
||||
* Use the initialize methods after construction.
|
||||
*/
|
||||
MessageReader() {
|
||||
}
|
||||
MessageReader() {}
|
||||
|
||||
/**
|
||||
* Returns a future which represents an attempt to read the next message in the child process's STDOUT. If the task
|
||||
|
|
@ -95,13 +94,10 @@ class MessageReader {
|
|||
* @param objectMapper The object mapper to decode messages.
|
||||
* @param executorService An executor service to run tasks in.
|
||||
*/
|
||||
MessageReader initialize(InputStream stream,
|
||||
String shardId,
|
||||
ObjectMapper objectMapper,
|
||||
ExecutorService executorService) {
|
||||
return this.initialize(new BufferedReader(new InputStreamReader(stream)), shardId, objectMapper,
|
||||
executorService);
|
||||
|
||||
MessageReader initialize(
|
||||
InputStream stream, String shardId, ObjectMapper objectMapper, ExecutorService executorService) {
|
||||
return this.initialize(
|
||||
new BufferedReader(new InputStreamReader(stream)), shardId, objectMapper, executorService);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -110,10 +106,8 @@ class MessageReader {
|
|||
* @param objectMapper The object mapper to decode messages.
|
||||
* @param executorService An executor service to run tasks in.
|
||||
*/
|
||||
MessageReader initialize(BufferedReader reader,
|
||||
String shardId,
|
||||
ObjectMapper objectMapper,
|
||||
ExecutorService executorService) {
|
||||
MessageReader initialize(
|
||||
BufferedReader reader, String shardId, ObjectMapper objectMapper, ExecutorService executorService) {
|
||||
this.reader = reader;
|
||||
this.shardId = shardId;
|
||||
this.objectMapper = objectMapper;
|
||||
|
|
|
|||
|
|
@ -23,7 +23,6 @@ import java.util.concurrent.ExecutorService;
|
|||
import java.util.concurrent.Future;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import software.amazon.kinesis.lifecycle.events.InitializationInput;
|
||||
import software.amazon.kinesis.lifecycle.events.LeaseLostInput;
|
||||
|
|
@ -55,8 +54,7 @@ class MessageWriter {
|
|||
/**
|
||||
* Use initialize method after construction.
|
||||
*/
|
||||
MessageWriter() {
|
||||
}
|
||||
MessageWriter() {}
|
||||
|
||||
/**
|
||||
* Writes the message then writes the line separator provided by the system. Flushes each message to guarantee it
|
||||
|
|
@ -76,7 +74,10 @@ class MessageWriter {
|
|||
*/
|
||||
synchronized (writer) {
|
||||
writer.write(message, 0, message.length());
|
||||
writer.write(System.lineSeparator(), 0, System.lineSeparator().length());
|
||||
writer.write(
|
||||
System.lineSeparator(),
|
||||
0,
|
||||
System.lineSeparator().length());
|
||||
writer.flush();
|
||||
}
|
||||
log.info("Message size == {} bytes for shard {}", message.getBytes().length, shardId);
|
||||
|
|
@ -108,9 +109,9 @@ class MessageWriter {
|
|||
String jsonText = objectMapper.writeValueAsString(message);
|
||||
return writeMessageToOutput(jsonText);
|
||||
} catch (IOException e) {
|
||||
String errorMessage =
|
||||
String.format("Encountered I/O error while writing %s action to subprocess", message.getClass()
|
||||
.getSimpleName());
|
||||
String errorMessage = String.format(
|
||||
"Encountered I/O error while writing %s action to subprocess",
|
||||
message.getClass().getSimpleName());
|
||||
log.error(errorMessage, e);
|
||||
throw new RuntimeException(errorMessage, e);
|
||||
}
|
||||
|
|
@ -175,8 +176,8 @@ class MessageWriter {
|
|||
* @param throwable
|
||||
* The exception that was thrown by a checkpoint attempt. Null if one didn't occur.
|
||||
*/
|
||||
Future<Boolean> writeCheckpointMessageWithError(String sequenceNumber, Long subSequenceNumber,
|
||||
Throwable throwable) {
|
||||
Future<Boolean> writeCheckpointMessageWithError(
|
||||
String sequenceNumber, Long subSequenceNumber, Throwable throwable) {
|
||||
return writeMessage(new CheckpointMessage(sequenceNumber, subSequenceNumber, throwable));
|
||||
}
|
||||
|
||||
|
|
@ -207,12 +208,10 @@ class MessageWriter {
|
|||
* @param objectMapper The object mapper to encode messages.
|
||||
* @param executorService An executor service to run tasks in.
|
||||
*/
|
||||
MessageWriter initialize(OutputStream stream,
|
||||
String shardId,
|
||||
ObjectMapper objectMapper,
|
||||
ExecutorService executorService) {
|
||||
return this.initialize(new BufferedWriter(new OutputStreamWriter(stream)), shardId, objectMapper,
|
||||
executorService);
|
||||
MessageWriter initialize(
|
||||
OutputStream stream, String shardId, ObjectMapper objectMapper, ExecutorService executorService) {
|
||||
return this.initialize(
|
||||
new BufferedWriter(new OutputStreamWriter(stream)), shardId, objectMapper, executorService);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -221,15 +220,12 @@ class MessageWriter {
|
|||
* @param objectMapper The object mapper to encode messages.
|
||||
* @param executorService An executor service to run tasks in.
|
||||
*/
|
||||
MessageWriter initialize(BufferedWriter writer,
|
||||
String shardId,
|
||||
ObjectMapper objectMapper,
|
||||
ExecutorService executorService) {
|
||||
MessageWriter initialize(
|
||||
BufferedWriter writer, String shardId, ObjectMapper objectMapper, ExecutorService executorService) {
|
||||
this.writer = writer;
|
||||
this.shardId = shardId;
|
||||
this.objectMapper = objectMapper;
|
||||
this.executorService = executorService;
|
||||
return this;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,20 +26,18 @@ import java.util.concurrent.Future;
|
|||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
|
||||
import ch.qos.logback.classic.LoggerContext;
|
||||
import ch.qos.logback.classic.joran.JoranConfigurator;
|
||||
import ch.qos.logback.core.joran.spi.JoranException;
|
||||
import com.beust.jcommander.JCommander;
|
||||
import com.beust.jcommander.Parameter;
|
||||
import lombok.Data;
|
||||
import lombok.experimental.Accessors;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.beust.jcommander.JCommander;
|
||||
import com.beust.jcommander.Parameter;
|
||||
|
||||
import ch.qos.logback.classic.LoggerContext;
|
||||
import ch.qos.logback.classic.joran.JoranConfigurator;
|
||||
import ch.qos.logback.core.joran.spi.JoranException;
|
||||
import lombok.Data;
|
||||
import lombok.experimental.Accessors;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import software.amazon.kinesis.coordinator.Scheduler;
|
||||
|
||||
/**
|
||||
|
|
@ -63,10 +61,10 @@ import software.amazon.kinesis.coordinator.Scheduler;
|
|||
* applicationName = PythonKCLSample
|
||||
*
|
||||
* # Users can change the credentials provider the KCL will use to retrieve credentials.
|
||||
* # The DefaultAWSCredentialsProviderChain checks several other providers, which is
|
||||
* # The DefaultCredentialsProvider checks several other providers, which is
|
||||
* # described here:
|
||||
* # http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html
|
||||
* AWSCredentialsProvider = DefaultAWSCredentialsProviderChain
|
||||
* # https://sdk.amazonaws.com/java/api/2.0.0-preview-11/software/amazon/awssdk/auth/credentials/DefaultCredentialsProvider.html
|
||||
* AwsCredentialsProvider = DefaultCredentialsProvider
|
||||
* </pre>
|
||||
*/
|
||||
@Slf4j
|
||||
|
|
@ -75,11 +73,14 @@ public class MultiLangDaemon {
|
|||
@Parameter
|
||||
List<String> parameters = new ArrayList<>();
|
||||
|
||||
@Parameter(names = { "-p", "--properties-file" }, description = "Properties file to be used with the KCL")
|
||||
@Parameter(
|
||||
names = {"-p", "--properties-file"},
|
||||
description = "Properties file to be used with the KCL")
|
||||
String propertiesFile;
|
||||
|
||||
@Parameter(names = { "-l",
|
||||
"--log-configuration" }, description = "File location of logback.xml to be override the default")
|
||||
@Parameter(
|
||||
names = {"-l", "--log-configuration"},
|
||||
description = "File location of logback.xml to be override the default")
|
||||
String logConfiguration;
|
||||
}
|
||||
|
||||
|
|
@ -102,7 +103,8 @@ public class MultiLangDaemon {
|
|||
}
|
||||
|
||||
JCommander buildJCommanderAndParseArgs(final MultiLangDaemonArguments arguments, final String[] args) {
|
||||
JCommander jCommander = JCommander.newBuilder().programName("amazon-kinesis-client MultiLangDaemon")
|
||||
JCommander jCommander = JCommander.newBuilder()
|
||||
.programName("amazon-kinesis-client MultiLangDaemon")
|
||||
.addObject(arguments)
|
||||
.build();
|
||||
jCommander.parse(args);
|
||||
|
|
@ -128,8 +130,8 @@ public class MultiLangDaemon {
|
|||
}
|
||||
}
|
||||
|
||||
void configureLogging(final String logConfiguration, final LoggerContext loggerContext,
|
||||
final JoranConfigurator configurator) {
|
||||
void configureLogging(
|
||||
final String logConfiguration, final LoggerContext loggerContext, final JoranConfigurator configurator) {
|
||||
loggerContext.reset();
|
||||
try (InputStream inputStream = FileUtils.openInputStream(new File(logConfiguration))) {
|
||||
configurator.setContext(loggerContext);
|
||||
|
|
@ -139,15 +141,14 @@ public class MultiLangDaemon {
|
|||
}
|
||||
}
|
||||
|
||||
String propertiesFile(final MultiLangDaemonArguments arguments) {
|
||||
String validateAndGetPropertiesFileName(final MultiLangDaemonArguments arguments) {
|
||||
String propertiesFile = "";
|
||||
|
||||
if (CollectionUtils.isNotEmpty(arguments.parameters)) {
|
||||
if (arguments.parameters.size() == 1) {
|
||||
propertiesFile = arguments.parameters.get(0);
|
||||
} else {
|
||||
throw new RuntimeException(
|
||||
"Expected a single argument, but found multiple arguments. Arguments: "
|
||||
throw new RuntimeException("Expected a single argument, but found multiple arguments. Arguments: "
|
||||
+ String.join(", ", arguments.parameters));
|
||||
}
|
||||
}
|
||||
|
|
@ -215,9 +216,9 @@ public class MultiLangDaemon {
|
|||
MultiLangDaemonArguments arguments = new MultiLangDaemonArguments();
|
||||
JCommander jCommander = daemon.buildJCommanderAndParseArgs(arguments, args);
|
||||
try {
|
||||
String propertiesFile = daemon.propertiesFile(arguments);
|
||||
String propertiesFileName = daemon.validateAndGetPropertiesFileName(arguments);
|
||||
daemon.configureLogging(arguments.logConfiguration);
|
||||
MultiLangDaemonConfig config = daemon.buildMultiLangDaemonConfig(propertiesFile);
|
||||
MultiLangDaemonConfig config = daemon.buildMultiLangDaemonConfig(propertiesFileName);
|
||||
|
||||
Scheduler scheduler = daemon.buildScheduler(config);
|
||||
MultiLangRunner runner = new MultiLangRunner(scheduler);
|
||||
|
|
|
|||
|
|
@ -26,10 +26,9 @@ import java.util.concurrent.SynchronousQueue;
|
|||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import software.amazon.kinesis.multilang.config.KinesisClientLibConfigurator;
|
||||
import com.google.common.util.concurrent.ThreadFactoryBuilder;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import software.amazon.kinesis.multilang.config.KinesisClientLibConfigurator;
|
||||
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
|
||||
import software.amazon.kinesis.retrieval.RetrievalConfig;
|
||||
|
||||
|
|
@ -45,11 +44,11 @@ public class MultiLangDaemonConfig {
|
|||
private static final String PROP_PROCESSING_LANGUAGE = "processingLanguage";
|
||||
private static final String PROP_MAX_ACTIVE_THREADS = "maxActiveThreads";
|
||||
|
||||
private MultiLangDaemonConfiguration multiLangDaemonConfiguration;
|
||||
private final MultiLangDaemonConfiguration multiLangDaemonConfiguration;
|
||||
|
||||
private ExecutorService executorService;
|
||||
private final ExecutorService executorService;
|
||||
|
||||
private MultiLangRecordProcessorFactory recordProcessorFactory;
|
||||
private final MultiLangRecordProcessorFactory recordProcessorFactory;
|
||||
|
||||
/**
|
||||
* Constructor.
|
||||
|
|
@ -94,8 +93,9 @@ public class MultiLangDaemonConfig {
|
|||
* @throws IllegalArgumentException
|
||||
* Thrown when the contents of the properties file are not as expected.
|
||||
*/
|
||||
public MultiLangDaemonConfig(String propertiesFile, ClassLoader classLoader,
|
||||
KinesisClientLibConfigurator configurator) throws IOException, IllegalArgumentException {
|
||||
public MultiLangDaemonConfig(
|
||||
String propertiesFile, ClassLoader classLoader, KinesisClientLibConfigurator configurator)
|
||||
throws IOException, IllegalArgumentException {
|
||||
Properties properties = loadProperties(classLoader, propertiesFile);
|
||||
if (!validateProperties(properties)) {
|
||||
throw new IllegalArgumentException(
|
||||
|
|
@ -107,11 +107,14 @@ public class MultiLangDaemonConfig {
|
|||
|
||||
multiLangDaemonConfiguration = configurator.getConfiguration(properties);
|
||||
executorService = buildExecutorService(properties);
|
||||
recordProcessorFactory = new MultiLangRecordProcessorFactory(executableName, executorService,
|
||||
multiLangDaemonConfiguration);
|
||||
recordProcessorFactory =
|
||||
new MultiLangRecordProcessorFactory(executableName, executorService, multiLangDaemonConfiguration);
|
||||
|
||||
log.info("Running {} to process stream {} with executable {}", multiLangDaemonConfiguration.getApplicationName(),
|
||||
multiLangDaemonConfiguration.getStreamName(), executableName);
|
||||
log.info(
|
||||
"Running {} to process stream {} with executable {}",
|
||||
multiLangDaemonConfiguration.getApplicationName(),
|
||||
multiLangDaemonConfiguration.getStreamName(),
|
||||
executableName);
|
||||
prepare(processingLanguage);
|
||||
}
|
||||
|
||||
|
|
@ -165,7 +168,6 @@ public class MultiLangDaemonConfig {
|
|||
propertyStream.close();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private static boolean validateProperties(Properties properties) {
|
||||
|
|
@ -182,12 +184,17 @@ public class MultiLangDaemonConfig {
|
|||
log.debug("Value for {} property is {}", PROP_MAX_ACTIVE_THREADS, maxActiveThreads);
|
||||
if (maxActiveThreads <= 0) {
|
||||
log.info("Using a cached thread pool.");
|
||||
return new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
|
||||
builder.build());
|
||||
return new ThreadPoolExecutor(
|
||||
0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, new SynchronousQueue<>(), builder.build());
|
||||
} else {
|
||||
log.info("Using a fixed thread pool with {} max active threads.", maxActiveThreads);
|
||||
return new ThreadPoolExecutor(maxActiveThreads, maxActiveThreads, 0L, TimeUnit.MILLISECONDS,
|
||||
new LinkedBlockingQueue<Runnable>(), builder.build());
|
||||
return new ThreadPoolExecutor(
|
||||
maxActiveThreads,
|
||||
maxActiveThreads,
|
||||
0L,
|
||||
TimeUnit.MILLISECONDS,
|
||||
new LinkedBlockingQueue<>(),
|
||||
builder.build());
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -61,8 +61,11 @@ class MultiLangProtocol {
|
|||
* @param initializationInput
|
||||
* information about the shard this processor is starting to process
|
||||
*/
|
||||
MultiLangProtocol(MessageReader messageReader, MessageWriter messageWriter,
|
||||
InitializationInput initializationInput, MultiLangDaemonConfiguration configuration) {
|
||||
MultiLangProtocol(
|
||||
MessageReader messageReader,
|
||||
MessageWriter messageWriter,
|
||||
InitializationInput initializationInput,
|
||||
MultiLangDaemonConfiguration configuration) {
|
||||
this.messageReader = messageReader;
|
||||
this.messageWriter = messageWriter;
|
||||
this.initializationInput = initializationInput;
|
||||
|
|
@ -82,7 +85,6 @@ class MultiLangProtocol {
|
|||
*/
|
||||
Future<Boolean> writeFuture = messageWriter.writeInitializeMessage(initializationInput);
|
||||
return waitForStatusMessage(InitializeMessage.ACTION, null, writeFuture);
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -115,7 +117,9 @@ class MultiLangProtocol {
|
|||
* @return
|
||||
*/
|
||||
boolean shardEnded(ShardEndedInput shardEndedInput) {
|
||||
return waitForStatusMessage(ShardEndedMessage.ACTION, shardEndedInput.checkpointer(),
|
||||
return waitForStatusMessage(
|
||||
ShardEndedMessage.ACTION,
|
||||
shardEndedInput.checkpointer(),
|
||||
messageWriter.writeShardEndedMessage(shardEndedInput));
|
||||
}
|
||||
|
||||
|
|
@ -147,8 +151,8 @@ class MultiLangProtocol {
|
|||
* The writing task.
|
||||
* @return Whether or not this operation succeeded.
|
||||
*/
|
||||
private boolean waitForStatusMessage(String action, RecordProcessorCheckpointer checkpointer,
|
||||
Future<Boolean> writeFuture) {
|
||||
private boolean waitForStatusMessage(
|
||||
String action, RecordProcessorCheckpointer checkpointer, Future<Boolean> writeFuture) {
|
||||
boolean statusWasCorrect = waitForStatusMessage(action, checkpointer);
|
||||
|
||||
// Examine whether or not we failed somewhere along the line.
|
||||
|
|
@ -207,13 +211,17 @@ class MultiLangProtocol {
|
|||
try {
|
||||
return Optional.of(fm.get());
|
||||
} catch (InterruptedException e) {
|
||||
log.error("Interrupted while waiting for {} message for shard {}", action,
|
||||
initializationInput.shardId(), e);
|
||||
log.error(
|
||||
"Interrupted while waiting for {} message for shard {}", action, initializationInput.shardId(), e);
|
||||
} catch (ExecutionException e) {
|
||||
log.error("Failed to get status message for {} action for shard {}", action,
|
||||
initializationInput.shardId(), e);
|
||||
log.error(
|
||||
"Failed to get status message for {} action for shard {}",
|
||||
action,
|
||||
initializationInput.shardId(),
|
||||
e);
|
||||
} catch (TimeoutException e) {
|
||||
log.error("Timedout to get status message for {} action for shard {}. Terminating...",
|
||||
log.error(
|
||||
"Timedout to get status message for {} action for shard {}. Terminating...",
|
||||
action,
|
||||
initializationInput.shardId(),
|
||||
e);
|
||||
|
|
@ -240,11 +248,14 @@ class MultiLangProtocol {
|
|||
* @return Whether or not this operation succeeded.
|
||||
*/
|
||||
private boolean validateStatusMessage(StatusMessage statusMessage, String action) {
|
||||
log.info("Received response {} from subprocess while waiting for {}"
|
||||
+ " while processing shard {}", statusMessage, action, initializationInput.shardId());
|
||||
return !(statusMessage == null || statusMessage.getResponseFor() == null || !statusMessage.getResponseFor()
|
||||
.equals(action));
|
||||
|
||||
log.info(
|
||||
"Received response {} from subprocess while waiting for {}" + " while processing shard {}",
|
||||
statusMessage,
|
||||
action,
|
||||
initializationInput.shardId());
|
||||
return !(statusMessage == null
|
||||
|| statusMessage.getResponseFor() == null
|
||||
|| !statusMessage.getResponseFor().equals(action));
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -274,13 +285,12 @@ class MultiLangProtocol {
|
|||
}
|
||||
return this.messageWriter.writeCheckpointMessageWithError(sequenceNumber, subSequenceNumber, null);
|
||||
} else {
|
||||
String message =
|
||||
String.format("Was asked to checkpoint at %s but no checkpointer was provided for shard %s",
|
||||
String message = String.format(
|
||||
"Was asked to checkpoint at %s but no checkpointer was provided for shard %s",
|
||||
sequenceNumber, initializationInput.shardId());
|
||||
log.error(message);
|
||||
return this.messageWriter.writeCheckpointMessageWithError(sequenceNumber, subSequenceNumber,
|
||||
new InvalidStateException(
|
||||
message));
|
||||
return this.messageWriter.writeCheckpointMessageWithError(
|
||||
sequenceNumber, subSequenceNumber, new InvalidStateException(message));
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
return this.messageWriter.writeCheckpointMessageWithError(sequenceNumber, subSequenceNumber, t);
|
||||
|
|
@ -288,8 +298,8 @@ class MultiLangProtocol {
|
|||
}
|
||||
|
||||
private String logCheckpointMessage(String sequenceNumber, Long subSequenceNumber) {
|
||||
return String.format("Attempting to checkpoint shard %s @ sequence number %s, and sub sequence number %s",
|
||||
return String.format(
|
||||
"Attempting to checkpoint shard %s @ sequence number %s, and sub sequence number %s",
|
||||
initializationInput.shardId(), sequenceNumber, subSequenceNumber);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,11 +17,10 @@ package software.amazon.kinesis.multilang;
|
|||
import java.util.concurrent.ExecutorService;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
|
||||
import software.amazon.kinesis.processor.ShardRecordProcessorFactory;
|
||||
import software.amazon.kinesis.processor.ShardRecordProcessor;
|
||||
import software.amazon.kinesis.processor.ShardRecordProcessorFactory;
|
||||
|
||||
/**
|
||||
* Creates {@link MultiLangShardRecordProcessor}'s.
|
||||
|
|
@ -43,8 +42,8 @@ public class MultiLangRecordProcessorFactory implements ShardRecordProcessorFact
|
|||
* @param command The command that will do processing for this factory's record processors.
|
||||
* @param executorService An executor service to use while processing inputs and outputs of the child process.
|
||||
*/
|
||||
public MultiLangRecordProcessorFactory(String command, ExecutorService executorService,
|
||||
MultiLangDaemonConfiguration configuration) {
|
||||
public MultiLangRecordProcessorFactory(
|
||||
String command, ExecutorService executorService, MultiLangDaemonConfiguration configuration) {
|
||||
this(command, executorService, new ObjectMapper(), configuration);
|
||||
}
|
||||
|
||||
|
|
@ -53,7 +52,10 @@ public class MultiLangRecordProcessorFactory implements ShardRecordProcessorFact
|
|||
* @param executorService An executor service to use while processing inputs and outputs of the child process.
|
||||
* @param objectMapper An object mapper used to convert messages to json to be written to the child process
|
||||
*/
|
||||
public MultiLangRecordProcessorFactory(String command, ExecutorService executorService, ObjectMapper objectMapper,
|
||||
public MultiLangRecordProcessorFactory(
|
||||
String command,
|
||||
ExecutorService executorService,
|
||||
ObjectMapper objectMapper,
|
||||
MultiLangDaemonConfiguration configuration) {
|
||||
this.command = command;
|
||||
this.commandArray = command.split(COMMAND_DELIMETER_REGEX);
|
||||
|
|
@ -68,8 +70,8 @@ public class MultiLangRecordProcessorFactory implements ShardRecordProcessorFact
|
|||
/*
|
||||
* Giving ProcessBuilder the command as an array of Strings allows users to specify command line arguments.
|
||||
*/
|
||||
return new MultiLangShardRecordProcessor(new ProcessBuilder(commandArray), executorService, this.objectMapper,
|
||||
this.configuration);
|
||||
return new MultiLangShardRecordProcessor(
|
||||
new ProcessBuilder(commandArray), executorService, this.objectMapper, this.configuration);
|
||||
}
|
||||
|
||||
String[] getCommandArray() {
|
||||
|
|
|
|||
|
|
@ -22,7 +22,6 @@ import java.util.concurrent.Future;
|
|||
import java.util.function.Function;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import software.amazon.kinesis.lifecycle.events.InitializationInput;
|
||||
import software.amazon.kinesis.lifecycle.events.LeaseLostInput;
|
||||
|
|
@ -32,7 +31,6 @@ import software.amazon.kinesis.lifecycle.events.ShutdownRequestedInput;
|
|||
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
|
||||
import software.amazon.kinesis.processor.ShardRecordProcessor;
|
||||
|
||||
|
||||
/**
|
||||
* A record processor that manages creating a child process that implements the multi language protocol and connecting
|
||||
* that child process's input and outputs to a {@link MultiLangProtocol} object and calling the appropriate methods on
|
||||
|
|
@ -50,20 +48,20 @@ public class MultiLangShardRecordProcessor implements ShardRecordProcessor {
|
|||
|
||||
private Future<?> stderrReadTask;
|
||||
|
||||
private MessageWriter messageWriter;
|
||||
private MessageReader messageReader;
|
||||
private DrainChildSTDERRTask readSTDERRTask;
|
||||
private final MessageWriter messageWriter;
|
||||
private final MessageReader messageReader;
|
||||
private final DrainChildSTDERRTask readSTDERRTask;
|
||||
|
||||
private ProcessBuilder processBuilder;
|
||||
private final ProcessBuilder processBuilder;
|
||||
private Process process;
|
||||
private ExecutorService executorService;
|
||||
private final ExecutorService executorService;
|
||||
private ProcessState state;
|
||||
|
||||
private ObjectMapper objectMapper;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
private MultiLangProtocol protocol;
|
||||
|
||||
private MultiLangDaemonConfiguration configuration;
|
||||
private final MultiLangDaemonConfiguration configuration;
|
||||
|
||||
@Override
|
||||
public void initialize(InitializationInput initializationInput) {
|
||||
|
|
@ -157,8 +155,10 @@ public class MultiLangShardRecordProcessor implements ShardRecordProcessor {
|
|||
if (ProcessState.ACTIVE.equals(this.state)) {
|
||||
stopProcessing("Encountered an error while trying to shutdown child process", t);
|
||||
} else {
|
||||
stopProcessing("Encountered an error during shutdown,"
|
||||
+ " but it appears the processor has already been shutdown", t);
|
||||
stopProcessing(
|
||||
"Encountered an error during shutdown,"
|
||||
+ " but it appears the processor has already been shutdown",
|
||||
t);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -167,7 +167,8 @@ public class MultiLangShardRecordProcessor implements ShardRecordProcessor {
|
|||
* Used to tell whether the processor has been shutdown already.
|
||||
*/
|
||||
private enum ProcessState {
|
||||
ACTIVE, SHUTDOWN
|
||||
ACTIVE,
|
||||
SHUTDOWN
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -180,10 +181,19 @@ public class MultiLangShardRecordProcessor implements ShardRecordProcessor {
|
|||
* @param objectMapper
|
||||
* An obejct mapper.
|
||||
*/
|
||||
MultiLangShardRecordProcessor(ProcessBuilder processBuilder, ExecutorService executorService,
|
||||
ObjectMapper objectMapper, MultiLangDaemonConfiguration configuration) {
|
||||
this(processBuilder, executorService, objectMapper, new MessageWriter(), new MessageReader(),
|
||||
new DrainChildSTDERRTask(), configuration);
|
||||
MultiLangShardRecordProcessor(
|
||||
ProcessBuilder processBuilder,
|
||||
ExecutorService executorService,
|
||||
ObjectMapper objectMapper,
|
||||
MultiLangDaemonConfiguration configuration) {
|
||||
this(
|
||||
processBuilder,
|
||||
executorService,
|
||||
objectMapper,
|
||||
new MessageWriter(),
|
||||
new MessageReader(),
|
||||
new DrainChildSTDERRTask(),
|
||||
configuration);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -202,8 +212,13 @@ public class MultiLangShardRecordProcessor implements ShardRecordProcessor {
|
|||
* @param readSTDERRTask
|
||||
* Error reader to read from child process's stderr
|
||||
*/
|
||||
MultiLangShardRecordProcessor(ProcessBuilder processBuilder, ExecutorService executorService, ObjectMapper objectMapper,
|
||||
MessageWriter messageWriter, MessageReader messageReader, DrainChildSTDERRTask readSTDERRTask,
|
||||
MultiLangShardRecordProcessor(
|
||||
ProcessBuilder processBuilder,
|
||||
ExecutorService executorService,
|
||||
ObjectMapper objectMapper,
|
||||
MessageWriter messageWriter,
|
||||
MessageReader messageReader,
|
||||
DrainChildSTDERRTask readSTDERRTask,
|
||||
MultiLangDaemonConfiguration configuration) {
|
||||
this.executorService = executorService;
|
||||
this.processBuilder = processBuilder;
|
||||
|
|
@ -213,7 +228,6 @@ public class MultiLangShardRecordProcessor implements ShardRecordProcessor {
|
|||
this.readSTDERRTask = readSTDERRTask;
|
||||
this.configuration = configuration;
|
||||
|
||||
|
||||
this.state = ProcessState.ACTIVE;
|
||||
}
|
||||
|
||||
|
|
@ -303,8 +317,6 @@ public class MultiLangShardRecordProcessor implements ShardRecordProcessor {
|
|||
|
||||
/**
|
||||
* We provide a package level method for unit testing this call to exit.
|
||||
*
|
||||
* @param val exit value
|
||||
*/
|
||||
void exit() {
|
||||
System.exit(EXIT_VALUE);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,148 @@
|
|||
/*
|
||||
* Copyright 2023 Amazon.com, Inc. or its affiliates.
|
||||
* Licensed under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package software.amazon.kinesis.multilang;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.google.common.base.CaseFormat;
|
||||
import lombok.AccessLevel;
|
||||
import lombok.Getter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import software.amazon.awssdk.regions.Region;
|
||||
|
||||
/**
|
||||
* Key-Value pairs which may be nested in, and extracted from, a property value
|
||||
* in a Java properties file. For example, given the line in a property file of
|
||||
* {@code my_key = my_value|foo=bar} and a delimiter split on {@code |} (pipe),
|
||||
* the value {@code my_value|foo=bar} would have a nested key of {@code foo}
|
||||
* and its corresponding value is {@code bar}.
|
||||
* <br/><br/>
|
||||
* The order of nested properties does not matter, and these properties are optional.
|
||||
* Customers may choose to provide, in any order, zero-or-more nested properties.
|
||||
* <br/><br/>
|
||||
* Duplicate keys are not supported, and may result in a last-write-wins outcome.
|
||||
*/
|
||||
@Slf4j
|
||||
public enum NestedPropertyKey {
|
||||
|
||||
/**
|
||||
* Specify the service endpoint where requests will be submitted.
|
||||
* This property's value must be in the following format:
|
||||
* <pre>
|
||||
* ENDPOINT ::= SERVICE_ENDPOINT "^" SIGNING_REGION
|
||||
* SERVICE_ENDPOINT ::= URL
|
||||
* SIGNING_REGION ::= AWS_REGION
|
||||
* </pre>
|
||||
*
|
||||
* It would be redundant to provide both this and {@link #ENDPOINT_REGION}.
|
||||
*
|
||||
* @see #ENDPOINT_REGION
|
||||
* @see <a href="https://docs.aws.amazon.com/general/latest/gr/rande.html">AWS Service endpoints</a>
|
||||
* @see <a href="https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-regions">Available Regions</a>
|
||||
*/
|
||||
ENDPOINT {
|
||||
void visit(final NestedPropertyProcessor processor, final String endpoint) {
|
||||
final String[] tokens = endpoint.split("\\^");
|
||||
if (tokens.length != 2) {
|
||||
throw new IllegalArgumentException("Invalid " + name() + ": " + endpoint);
|
||||
}
|
||||
processor.acceptEndpoint(tokens[0], tokens[1]);
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Specify the region where service requests will be submitted. This
|
||||
* region will determine both the service endpoint and signing region.
|
||||
* <br/><br/>
|
||||
* It would be redundant to provide both this and {@link #ENDPOINT}.
|
||||
*
|
||||
* @see #ENDPOINT
|
||||
* @see <a href="https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-regions">Available Regions</a>
|
||||
*/
|
||||
ENDPOINT_REGION {
|
||||
void visit(final NestedPropertyProcessor processor, final String regionName) {
|
||||
List<Region> validRegions = Region.regions();
|
||||
Region region = Region.of(regionName);
|
||||
if (!validRegions.contains(region)) {
|
||||
throw new IllegalArgumentException("Invalid region name: " + regionName);
|
||||
}
|
||||
processor.acceptEndpointRegion(region);
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* External ids may be used when delegating access in a multi-tenant
|
||||
* environment, or to third parties.
|
||||
*
|
||||
* @see <a href="https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-user_externalid.html">
|
||||
* How to use an external ID when granting access to your AWS resources to a third party</a>
|
||||
*/
|
||||
EXTERNAL_ID {
|
||||
void visit(final NestedPropertyProcessor processor, final String externalId) {
|
||||
processor.acceptExternalId(externalId);
|
||||
}
|
||||
},
|
||||
;
|
||||
|
||||
/**
|
||||
* Nested key within the property value. For example, a nested key-value
|
||||
* of {@code foo=bar} has a nested key of {@code foo}.
|
||||
*/
|
||||
@Getter(AccessLevel.PACKAGE)
|
||||
private final String nestedKey;
|
||||
|
||||
NestedPropertyKey() {
|
||||
// convert the enum from UPPER_SNAKE_CASE to lowerCamelCase
|
||||
nestedKey = CaseFormat.UPPER_UNDERSCORE.to(CaseFormat.LOWER_CAMEL, name());
|
||||
}
|
||||
|
||||
abstract void visit(NestedPropertyProcessor processor, String value);
|
||||
|
||||
/**
|
||||
* Parses any number of parameters. Each nested property will prompt a
|
||||
* visit to the {@code processor}.
|
||||
*
|
||||
* @param processor processor to be invoked for every nested property
|
||||
* @param params parameters to check for a nested property key
|
||||
*/
|
||||
public static void parse(final NestedPropertyProcessor processor, final String... params) {
|
||||
// Construct a disposable cache to keep this O(n). Since parsing is
|
||||
// usually one-and-done, it's wasteful to maintain this cache in perpetuity.
|
||||
final Map<String, NestedPropertyKey> cachedKeys = new HashMap<>();
|
||||
for (final NestedPropertyKey npk : values()) {
|
||||
cachedKeys.put(npk.getNestedKey(), npk);
|
||||
}
|
||||
|
||||
for (final String param : params) {
|
||||
if (param != null) {
|
||||
final String[] tokens = param.split("=");
|
||||
if (tokens.length == 2) {
|
||||
final NestedPropertyKey npk = cachedKeys.get(tokens[0]);
|
||||
if (npk != null) {
|
||||
npk.visit(processor, tokens[1]);
|
||||
} else {
|
||||
log.warn("Unsupported nested key: {}", param);
|
||||
}
|
||||
} else if (tokens.length > 2) {
|
||||
log.warn("Malformed nested key: {}", param);
|
||||
} else {
|
||||
log.info("Parameter is not a nested key: {}", param);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
/*
|
||||
* Copyright 2023 Amazon.com, Inc. or its affiliates.
|
||||
* Licensed under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package software.amazon.kinesis.multilang;
|
||||
|
||||
import software.amazon.awssdk.regions.Region;
|
||||
|
||||
/**
|
||||
* Defines methods to process {@link NestedPropertyKey}s.
|
||||
*/
|
||||
public interface NestedPropertyProcessor {
|
||||
|
||||
/**
|
||||
* Set the service endpoint where requests are sent.
|
||||
*
|
||||
* @param serviceEndpoint the service endpoint either with or without the protocol
|
||||
* (e.g., https://sns.us-west-1.amazonaws.com, sns.us-west-1.amazonaws.com)
|
||||
* @param signingRegion the region to use for the client (e.g. us-west-1)
|
||||
*
|
||||
* @see #acceptEndpointRegion(Region)
|
||||
* @see <a href="https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/core/client/builder/SdkClientBuilder.html#endpointOverride(java.net.URI)">
|
||||
* AwsClientBuilder.endpointOverride</a>
|
||||
*/
|
||||
void acceptEndpoint(String serviceEndpoint, String signingRegion);
|
||||
|
||||
/**
|
||||
* Set the service endpoint where requests are sent.
|
||||
*
|
||||
* @param region Region to be used by the client. This will be used to determine both the service endpoint
|
||||
* (e.g., https://sns.us-west-1.amazonaws.com) and signing region (e.g., us-west-1) for requests.
|
||||
*
|
||||
* @see #acceptEndpoint(String, String)
|
||||
*/
|
||||
void acceptEndpointRegion(Region region);
|
||||
|
||||
/**
|
||||
* Set the external id, an optional field to designate who can assume an IAM role.
|
||||
*
|
||||
* @param externalId external id used in the service call used to retrieve session credentials
|
||||
*/
|
||||
void acceptExternalId(String externalId);
|
||||
}
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
/*
|
||||
* Copyright 2024 Amazon.com, Inc. or its affiliates.
|
||||
* Licensed under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package software.amazon.kinesis.multilang.auth;
|
||||
|
||||
import java.net.URI;
|
||||
import java.util.Arrays;
|
||||
|
||||
import software.amazon.awssdk.auth.credentials.AwsCredentials;
|
||||
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
|
||||
import software.amazon.awssdk.regions.Region;
|
||||
import software.amazon.awssdk.services.sts.StsClient;
|
||||
import software.amazon.awssdk.services.sts.StsClientBuilder;
|
||||
import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider;
|
||||
import software.amazon.awssdk.services.sts.model.AssumeRoleRequest;
|
||||
import software.amazon.awssdk.services.sts.model.AssumeRoleRequest.Builder;
|
||||
import software.amazon.kinesis.multilang.NestedPropertyKey;
|
||||
import software.amazon.kinesis.multilang.NestedPropertyProcessor;
|
||||
|
||||
public class KclStsAssumeRoleCredentialsProvider implements AwsCredentialsProvider, NestedPropertyProcessor {
|
||||
private final Builder assumeRoleRequestBuilder;
|
||||
private final StsClientBuilder stsClientBuilder;
|
||||
private final StsAssumeRoleCredentialsProvider stsAssumeRoleCredentialsProvider;
|
||||
|
||||
public KclStsAssumeRoleCredentialsProvider(String[] params) {
|
||||
this(params[0], params[1], Arrays.copyOfRange(params, 2, params.length));
|
||||
}
|
||||
|
||||
public KclStsAssumeRoleCredentialsProvider(String roleArn, String roleSessionName, String... params) {
|
||||
this.assumeRoleRequestBuilder =
|
||||
AssumeRoleRequest.builder().roleArn(roleArn).roleSessionName(roleSessionName);
|
||||
this.stsClientBuilder = StsClient.builder();
|
||||
NestedPropertyKey.parse(this, params);
|
||||
this.stsAssumeRoleCredentialsProvider = StsAssumeRoleCredentialsProvider.builder()
|
||||
.refreshRequest(assumeRoleRequestBuilder.build())
|
||||
.asyncCredentialUpdateEnabled(true)
|
||||
.stsClient(stsClientBuilder.build())
|
||||
.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public AwsCredentials resolveCredentials() {
|
||||
return stsAssumeRoleCredentialsProvider.resolveCredentials();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void acceptEndpoint(String serviceEndpoint, String signingRegion) {
|
||||
if (!serviceEndpoint.startsWith("http://") && !serviceEndpoint.startsWith("https://")) {
|
||||
serviceEndpoint = "https://" + serviceEndpoint;
|
||||
}
|
||||
stsClientBuilder.endpointOverride(URI.create(serviceEndpoint));
|
||||
stsClientBuilder.region(Region.of(signingRegion));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void acceptEndpointRegion(Region region) {
|
||||
stsClientBuilder.region(region);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void acceptExternalId(String externalId) {
|
||||
assumeRoleRequestBuilder.externalId(externalId);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,129 +0,0 @@
|
|||
/*
|
||||
* Copyright 2019 Amazon.com, Inc. or its affiliates.
|
||||
* Licensed under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package software.amazon.kinesis.multilang.config;
|
||||
|
||||
import java.lang.reflect.Constructor;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import com.amazonaws.auth.AWSCredentialsProvider;
|
||||
import com.amazonaws.auth.AWSCredentialsProviderChain;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* Get AWSCredentialsProvider property.
|
||||
*/
|
||||
@Slf4j
|
||||
class AWSCredentialsProviderPropertyValueDecoder implements IPropertyValueDecoder<AWSCredentialsProvider> {
|
||||
private static final String AUTH_PREFIX = "com.amazonaws.auth.";
|
||||
private static final String LIST_DELIMITER = ",";
|
||||
private static final String ARG_DELIMITER = "|";
|
||||
|
||||
/**
|
||||
* Constructor.
|
||||
*/
|
||||
AWSCredentialsProviderPropertyValueDecoder() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Get AWSCredentialsProvider property.
|
||||
*
|
||||
* @param value
|
||||
* property value as String
|
||||
* @return corresponding variable in correct type
|
||||
*/
|
||||
@Override
|
||||
public AWSCredentialsProvider decodeValue(String value) {
|
||||
if (value != null) {
|
||||
List<String> providerNames = getProviderNames(value);
|
||||
List<AWSCredentialsProvider> providers = getValidCredentialsProviders(providerNames);
|
||||
AWSCredentialsProvider[] ps = new AWSCredentialsProvider[providers.size()];
|
||||
providers.toArray(ps);
|
||||
return new AWSCredentialsProviderChain(providers);
|
||||
} else {
|
||||
throw new IllegalArgumentException("Property AWSCredentialsProvider is missing.");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return list of supported types
|
||||
*/
|
||||
@Override
|
||||
public List<Class<AWSCredentialsProvider>> getSupportedTypes() {
|
||||
return Arrays.asList(AWSCredentialsProvider.class);
|
||||
}
|
||||
|
||||
/*
|
||||
* Convert string list to a list of valid credentials providers.
|
||||
*/
|
||||
private static List<AWSCredentialsProvider> getValidCredentialsProviders(List<String> providerNames) {
|
||||
List<AWSCredentialsProvider> credentialsProviders = new ArrayList<>();
|
||||
for (String providerName : providerNames) {
|
||||
if (providerName.contains(ARG_DELIMITER)) {
|
||||
String[] nameAndArgs = providerName.split("\\" + ARG_DELIMITER);
|
||||
Class<?>[] argTypes = new Class<?>[nameAndArgs.length - 1];
|
||||
Arrays.fill(argTypes, String.class);
|
||||
try {
|
||||
Class<?> className = Class.forName(nameAndArgs[0]);
|
||||
Constructor<?> c = className.getConstructor(argTypes);
|
||||
credentialsProviders.add((AWSCredentialsProvider) c
|
||||
.newInstance(Arrays.copyOfRange(nameAndArgs, 1, nameAndArgs.length)));
|
||||
} catch (Exception e) {
|
||||
log.debug("Can't find any credentials provider matching {}.", providerName);
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
Class<?> className = Class.forName(providerName);
|
||||
credentialsProviders.add((AWSCredentialsProvider) className.newInstance());
|
||||
} catch (Exception e) {
|
||||
log.debug("Can't find any credentials provider matching {}.", providerName);
|
||||
}
|
||||
}
|
||||
}
|
||||
return credentialsProviders;
|
||||
}
|
||||
|
||||
private static List<String> getProviderNames(String property) {
|
||||
// assume list delimiter is ","
|
||||
String[] elements = property.split(LIST_DELIMITER);
|
||||
List<String> result = new ArrayList<String>();
|
||||
for (int i = 0; i < elements.length; i++) {
|
||||
String string = elements[i].trim();
|
||||
if (!string.isEmpty()) {
|
||||
// find all possible names and add them to name list
|
||||
result.addAll(getPossibleFullClassNames(string));
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static List<String> getPossibleFullClassNames(String s) {
|
||||
/*
|
||||
* We take care of three cases :
|
||||
*
|
||||
* 1. Customer provides a short name of common providers in com.amazonaws.auth package i.e. any classes
|
||||
* implementing the AWSCredentialsProvider interface:
|
||||
* http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/AWSCredentialsProvider.html
|
||||
*
|
||||
* 2. Customer provides a full name of common providers e.g. com.amazonaws.auth.ClasspathFileCredentialsProvider
|
||||
*
|
||||
* 3. Customer provides a custom credentials provider with full name of provider
|
||||
*/
|
||||
|
||||
return Arrays.asList(s, AUTH_PREFIX + s);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,261 @@
|
|||
/*
|
||||
* Copyright 2019 Amazon.com, Inc. or its affiliates.
|
||||
* Licensed under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package software.amazon.kinesis.multilang.config;
|
||||
|
||||
import java.lang.reflect.InvocationTargetException;
|
||||
import java.lang.reflect.Method;
|
||||
import java.lang.reflect.Modifier;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
|
||||
import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain;
|
||||
import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider;
|
||||
import software.amazon.kinesis.multilang.auth.KclStsAssumeRoleCredentialsProvider;
|
||||
|
||||
/**
|
||||
* Get AwsCredentialsProvider property.
|
||||
*/
|
||||
@Slf4j
|
||||
class AwsCredentialsProviderPropertyValueDecoder implements IPropertyValueDecoder<AwsCredentialsProvider> {
|
||||
private static final String LIST_DELIMITER = ",";
|
||||
private static final String ARG_DELIMITER = "|";
|
||||
|
||||
/**
|
||||
* Constructor.
|
||||
*/
|
||||
AwsCredentialsProviderPropertyValueDecoder() {}
|
||||
|
||||
/**
|
||||
* Get AwsCredentialsProvider property.
|
||||
*
|
||||
* @param value
|
||||
* property value as String
|
||||
* @return corresponding variable in correct type
|
||||
*/
|
||||
@Override
|
||||
public AwsCredentialsProvider decodeValue(String value) {
|
||||
if (value != null) {
|
||||
List<String> providerNames = getProviderNames(value);
|
||||
List<AwsCredentialsProvider> providers = getValidCredentialsProviders(providerNames);
|
||||
AwsCredentialsProvider[] ps = new AwsCredentialsProvider[providers.size()];
|
||||
providers.toArray(ps);
|
||||
if (providers.isEmpty()) {
|
||||
log.warn("Unable to construct any provider with name {}", value);
|
||||
log.warn("Please verify that all AwsCredentialsProvider properties are passed correctly");
|
||||
}
|
||||
return AwsCredentialsProviderChain.builder()
|
||||
.credentialsProviders(providers)
|
||||
.build();
|
||||
} else {
|
||||
throw new IllegalArgumentException("Property AwsCredentialsProvider is missing.");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return list of supported types
|
||||
*/
|
||||
@Override
|
||||
public List<Class<AwsCredentialsProvider>> getSupportedTypes() {
|
||||
return Collections.singletonList(AwsCredentialsProvider.class);
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert string list to a list of valid credentials providers.
|
||||
*/
|
||||
private static List<AwsCredentialsProvider> getValidCredentialsProviders(List<String> providerNames) {
|
||||
List<AwsCredentialsProvider> credentialsProviders = new ArrayList<>();
|
||||
|
||||
for (String providerName : providerNames) {
|
||||
final String[] nameAndArgs = providerName.split("\\" + ARG_DELIMITER);
|
||||
final Class<? extends AwsCredentialsProvider> clazz = getClass(nameAndArgs[0]);
|
||||
if (clazz == null) {
|
||||
continue;
|
||||
}
|
||||
log.info("Attempting to construct {}", clazz);
|
||||
final String[] varargs =
|
||||
nameAndArgs.length > 1 ? Arrays.copyOfRange(nameAndArgs, 1, nameAndArgs.length) : new String[0];
|
||||
AwsCredentialsProvider provider = tryConstructor(providerName, clazz, varargs);
|
||||
if (provider == null) {
|
||||
provider = tryCreate(providerName, clazz, varargs);
|
||||
}
|
||||
if (provider != null) {
|
||||
log.info("Provider constructed successfully: {}", provider);
|
||||
credentialsProviders.add(provider);
|
||||
}
|
||||
}
|
||||
return credentialsProviders;
|
||||
}
|
||||
|
||||
private static AwsCredentialsProvider tryConstructor(
|
||||
String providerName, Class<? extends AwsCredentialsProvider> clazz, String[] varargs) {
|
||||
AwsCredentialsProvider provider =
|
||||
constructProvider(providerName, () -> getConstructorWithVarArgs(clazz, varargs));
|
||||
if (provider == null) {
|
||||
provider = constructProvider(providerName, () -> getConstructorWithArgs(clazz, varargs));
|
||||
}
|
||||
if (provider == null) {
|
||||
provider = constructProvider(providerName, clazz::newInstance);
|
||||
}
|
||||
return provider;
|
||||
}
|
||||
|
||||
private static AwsCredentialsProvider tryCreate(
|
||||
String providerName, Class<? extends AwsCredentialsProvider> clazz, String[] varargs) {
|
||||
AwsCredentialsProvider provider =
|
||||
constructProvider(providerName, () -> getCreateMethod(clazz, (Object) varargs));
|
||||
if (provider == null) {
|
||||
provider = constructProvider(providerName, () -> getCreateMethod(clazz, varargs));
|
||||
}
|
||||
if (provider == null) {
|
||||
provider = constructProvider(providerName, () -> getCreateMethod(clazz));
|
||||
}
|
||||
return provider;
|
||||
}
|
||||
|
||||
private static AwsCredentialsProvider getConstructorWithVarArgs(
|
||||
Class<? extends AwsCredentialsProvider> clazz, String[] varargs) {
|
||||
try {
|
||||
return clazz.getConstructor(String[].class).newInstance((Object) varargs);
|
||||
} catch (Exception e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static AwsCredentialsProvider getConstructorWithArgs(
|
||||
Class<? extends AwsCredentialsProvider> clazz, String[] varargs) {
|
||||
try {
|
||||
Class<?>[] argTypes = new Class<?>[varargs.length];
|
||||
Arrays.fill(argTypes, String.class);
|
||||
return clazz.getConstructor(argTypes).newInstance((Object[]) varargs);
|
||||
} catch (Exception e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static AwsCredentialsProvider getCreateMethod(
|
||||
Class<? extends AwsCredentialsProvider> clazz, Object... args) {
|
||||
try {
|
||||
Class<?>[] argTypes = new Class<?>[args.length];
|
||||
for (int i = 0; i < args.length; i++) {
|
||||
argTypes[i] = args[i].getClass();
|
||||
}
|
||||
Method createMethod = clazz.getDeclaredMethod("create", argTypes);
|
||||
if (Modifier.isStatic(createMethod.getModifiers())) {
|
||||
return clazz.cast(createMethod.invoke(null, args));
|
||||
} else {
|
||||
log.warn("Found non-static create() method in {}", clazz.getName());
|
||||
}
|
||||
} catch (NoSuchMethodException e) {
|
||||
// No matching create method found for class
|
||||
} catch (Exception e) {
|
||||
log.warn("Failed to invoke create() method in {}", clazz.getName(), e);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the class for the given provider name.
|
||||
*
|
||||
* @param providerName A string containing the provider name.
|
||||
*
|
||||
* @return The Class object representing the resolved AwsCredentialsProvider implementation,
|
||||
* or null if the class cannot be resolved or does not extend AwsCredentialsProvider.
|
||||
*/
|
||||
private static Class<? extends AwsCredentialsProvider> getClass(String providerName) {
|
||||
// Convert any form of StsAssumeRoleCredentialsProvider string to KclStsAssumeRoleCredentialsProvider
|
||||
if (providerName.equals(StsAssumeRoleCredentialsProvider.class.getSimpleName())
|
||||
|| providerName.equals(StsAssumeRoleCredentialsProvider.class.getName())) {
|
||||
providerName = KclStsAssumeRoleCredentialsProvider.class.getName();
|
||||
}
|
||||
try {
|
||||
final Class<?> c = Class.forName(providerName);
|
||||
if (!AwsCredentialsProvider.class.isAssignableFrom(c)) {
|
||||
return null;
|
||||
}
|
||||
return (Class<? extends AwsCredentialsProvider>) c;
|
||||
} catch (ClassNotFoundException cnfe) {
|
||||
// Providers are a product of prefixed Strings to cover multiple
|
||||
// namespaces (e.g., "Foo" -> { "some.auth.Foo", "kcl.auth.Foo" }).
|
||||
// It's expected that many class names will not resolve.
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static List<String> getProviderNames(String property) {
|
||||
// assume list delimiter is ","
|
||||
String[] elements = property.split(LIST_DELIMITER);
|
||||
List<String> result = new ArrayList<>();
|
||||
for (int i = 0; i < elements.length; i++) {
|
||||
String string = elements[i].trim();
|
||||
if (!string.isEmpty()) {
|
||||
// find all possible names and add them to name list
|
||||
result.addAll(getPossibleFullClassNames(string));
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static List<String> getPossibleFullClassNames(final String provider) {
|
||||
return Stream.of(
|
||||
// Customer provides a short name of a provider offered by this multi-lang package
|
||||
"software.amazon.kinesis.multilang.auth.",
|
||||
// Customer provides a short name of common providers in software.amazon.awssdk.auth.credentials
|
||||
// package (e.g., any classes implementing the AwsCredentialsProvider interface)
|
||||
// @see
|
||||
// https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/auth/credentials/AwsCredentialsProvider.html
|
||||
"software.amazon.awssdk.auth.credentials.",
|
||||
// Customer provides a fully-qualified provider name, or a custom credentials provider
|
||||
// (e.g., org.mycompany.FooProvider)
|
||||
"")
|
||||
.map(prefix -> prefix + provider)
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@FunctionalInterface
|
||||
private interface CredentialsProviderConstructor<T extends AwsCredentialsProvider> {
|
||||
T construct()
|
||||
throws IllegalAccessException, InstantiationException, InvocationTargetException, NoSuchMethodException;
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempts to construct an {@link AwsCredentialsProvider}.
|
||||
*
|
||||
* @param providerName Raw, unmodified provider name. Should there be an
|
||||
* Exception during construction, this parameter will be logged.
|
||||
* @param constructor supplier-like function that will perform the construction
|
||||
* @return the constructed provider, if successful; otherwise, null
|
||||
*
|
||||
* @param <T> type of the CredentialsProvider to construct
|
||||
*/
|
||||
private static <T extends AwsCredentialsProvider> T constructProvider(
|
||||
final String providerName, final CredentialsProviderConstructor<T> constructor) {
|
||||
try {
|
||||
return constructor.construct();
|
||||
} catch (NoSuchMethodException
|
||||
| IllegalAccessException
|
||||
| InstantiationException
|
||||
| InvocationTargetException
|
||||
| RuntimeException ignored) {
|
||||
// ignore
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
@ -51,16 +51,22 @@ public class BuilderDynaBean implements DynaBean {
|
|||
this(destinedClass, convertUtilsBean, null, Arrays.asList(classPrefixSearchList));
|
||||
}
|
||||
|
||||
public BuilderDynaBean(Class<?> destinedClass, ConvertUtilsBean convertUtilsBean,
|
||||
Function<String, ?> emptyPropertyHandler, String... classPrefixSearchList) {
|
||||
public BuilderDynaBean(
|
||||
Class<?> destinedClass,
|
||||
ConvertUtilsBean convertUtilsBean,
|
||||
Function<String, ?> emptyPropertyHandler,
|
||||
String... classPrefixSearchList) {
|
||||
this(destinedClass, convertUtilsBean, emptyPropertyHandler, Arrays.asList(classPrefixSearchList));
|
||||
}
|
||||
|
||||
public BuilderDynaBean(Class<?> destinedClass, ConvertUtilsBean convertUtilsBean,
|
||||
Function<String, ?> emtpyPropertyHandler, List<String> classPrefixSearchList) {
|
||||
public BuilderDynaBean(
|
||||
Class<?> destinedClass,
|
||||
ConvertUtilsBean convertUtilsBean,
|
||||
Function<String, ?> emptyPropertyHandler,
|
||||
List<String> classPrefixSearchList) {
|
||||
this.convertUtilsBean = convertUtilsBean;
|
||||
this.classPrefixSearchList = classPrefixSearchList;
|
||||
this.emptyPropertyHandler = emtpyPropertyHandler;
|
||||
this.emptyPropertyHandler = emptyPropertyHandler;
|
||||
initialize(destinedClass);
|
||||
}
|
||||
|
||||
|
|
@ -102,7 +108,6 @@ public class BuilderDynaBean implements DynaBean {
|
|||
// Ignored
|
||||
//
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -214,8 +219,10 @@ public class BuilderDynaBean implements DynaBean {
|
|||
validateCanBuildOrCreate();
|
||||
List<TypeTag> types = dynaBeanBuilderSupport.getProperty(name);
|
||||
if (types.size() > 1) {
|
||||
Optional<TypeTag> arrayType = types.stream().filter(t -> t.type.isArray()).findFirst();
|
||||
return arrayType.map(t -> new DynaProperty(name, t.type, t.type.getComponentType()))
|
||||
Optional<TypeTag> arrayType =
|
||||
types.stream().filter(t -> t.type.isArray()).findFirst();
|
||||
return arrayType
|
||||
.map(t -> new DynaProperty(name, t.type, t.type.getComponentType()))
|
||||
.orElseGet(() -> new DynaProperty(name));
|
||||
} else {
|
||||
TypeTag type = types.get(0);
|
||||
|
|
@ -232,7 +239,8 @@ public class BuilderDynaBean implements DynaBean {
|
|||
@Override
|
||||
public DynaProperty[] getDynaProperties() {
|
||||
validateCanBuildOrCreate();
|
||||
return dynaBeanBuilderSupport.getPropertyNames().stream().map(this::getDynaProperty)
|
||||
return dynaBeanBuilderSupport.getPropertyNames().stream()
|
||||
.map(this::getDynaProperty)
|
||||
.toArray(DynaProperty[]::new);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -23,13 +23,11 @@ import java.util.Map;
|
|||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
|
||||
import com.google.common.base.Defaults;
|
||||
import lombok.NonNull;
|
||||
import org.apache.commons.lang3.ClassUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import com.google.common.base.Defaults;
|
||||
|
||||
import lombok.NonNull;
|
||||
|
||||
public class ConfigurationSettableUtils {
|
||||
|
||||
public static <T> T resolveFields(@NonNull Object source, @NonNull T configObject) {
|
||||
|
|
@ -40,8 +38,8 @@ public class ConfigurationSettableUtils {
|
|||
return configObject;
|
||||
}
|
||||
|
||||
public static void resolveFields(Object source, Map<Class<?>, Object> configObjects, Set<Class<?>> restrictTo,
|
||||
Set<Class<?>> skipIf) {
|
||||
public static void resolveFields(
|
||||
Object source, Map<Class<?>, Object> configObjects, Set<Class<?>> restrictTo, Set<Class<?>> skipIf) {
|
||||
for (Field field : source.getClass().getDeclaredFields()) {
|
||||
for (ConfigurationSettable b : field.getAnnotationsByType(ConfigurationSettable.class)) {
|
||||
if (restrictTo != null && !restrictTo.contains(b.configurationClass())) {
|
||||
|
|
@ -70,9 +68,11 @@ public class ConfigurationSettableUtils {
|
|||
value = Optional.of(value);
|
||||
}
|
||||
if (ClassUtils.isPrimitiveOrWrapper(value.getClass())) {
|
||||
Class<?> primitiveType = field.getType().isPrimitive() ? field.getType()
|
||||
Class<?> primitiveType = field.getType().isPrimitive()
|
||||
? field.getType()
|
||||
: ClassUtils.wrapperToPrimitive(field.getType());
|
||||
Class<?> wrapperType = !field.getType().isPrimitive() ? field.getType()
|
||||
Class<?> wrapperType = !field.getType().isPrimitive()
|
||||
? field.getType()
|
||||
: ClassUtils.primitiveToWrapper(field.getType());
|
||||
|
||||
try {
|
||||
|
|
@ -93,9 +93,22 @@ public class ConfigurationSettableUtils {
|
|||
try {
|
||||
setter = b.configurationClass().getMethod(setterName, value.getClass());
|
||||
} catch (NoSuchMethodException e) {
|
||||
// find if there is a setter which is not the exact parameter type
|
||||
// but is assignable from the type
|
||||
for (Method method : b.configurationClass().getMethods()) {
|
||||
Class<?>[] parameterTypes = method.getParameterTypes();
|
||||
if (method.getName().equals(setterName)
|
||||
&& parameterTypes.length == 1
|
||||
&& parameterTypes[0].isAssignableFrom(value.getClass())) {
|
||||
setter = method;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (setter == null) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
try {
|
||||
setter.invoke(configObject, value);
|
||||
} catch (IllegalAccessException | InvocationTargetException e) {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,82 @@
|
|||
/*
|
||||
* Copyright 2024 Amazon.com, Inc. or its affiliates.
|
||||
* Licensed under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package software.amazon.kinesis.multilang.config;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import software.amazon.awssdk.services.dynamodb.model.BillingMode;
|
||||
import software.amazon.kinesis.coordinator.CoordinatorConfig.CoordinatorStateTableConfig;
|
||||
import software.amazon.kinesis.multilang.config.converter.TagConverter.TagCollection;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
public class CoordinatorStateTableConfigBean {
|
||||
|
||||
interface CoordinatorStateConfigBeanDelegate {
|
||||
String getCoordinatorStateTableName();
|
||||
|
||||
void setCoordinatorStateTableName(String value);
|
||||
|
||||
BillingMode getCoordinatorStateBillingMode();
|
||||
|
||||
void setCoordinatorStateBillingMode(BillingMode value);
|
||||
|
||||
long getCoordinatorStateReadCapacity();
|
||||
|
||||
void setCoordinatorStateReadCapacity(long value);
|
||||
|
||||
long getCoordinatorStateWriteCapacity();
|
||||
|
||||
void setCoordinatorStateWriteCapacity(long value);
|
||||
|
||||
Boolean getCoordinatorStatePointInTimeRecoveryEnabled();
|
||||
|
||||
void setCoordinatorStatePointInTimeRecoveryEnabled(Boolean value);
|
||||
|
||||
Boolean getCoordinatorStateDeletionProtectionEnabled();
|
||||
|
||||
void setCoordinatorStateDeletionProtectionEnabled(Boolean value);
|
||||
|
||||
TagCollection getCoordinatorStateTags();
|
||||
|
||||
void setCoordinatorStateTags(TagCollection value);
|
||||
}
|
||||
|
||||
@ConfigurationSettable(configurationClass = CoordinatorStateTableConfig.class, methodName = "tableName")
|
||||
private String coordinatorStateTableName;
|
||||
|
||||
@ConfigurationSettable(configurationClass = CoordinatorStateTableConfig.class, methodName = "billingMode")
|
||||
private BillingMode coordinatorStateBillingMode;
|
||||
|
||||
@ConfigurationSettable(configurationClass = CoordinatorStateTableConfig.class, methodName = "readCapacity")
|
||||
private long coordinatorStateReadCapacity;
|
||||
|
||||
@ConfigurationSettable(configurationClass = CoordinatorStateTableConfig.class, methodName = "writeCapacity")
|
||||
private long coordinatorStateWriteCapacity;
|
||||
|
||||
@ConfigurationSettable(
|
||||
configurationClass = CoordinatorStateTableConfig.class,
|
||||
methodName = "pointInTimeRecoveryEnabled")
|
||||
private Boolean coordinatorStatePointInTimeRecoveryEnabled;
|
||||
|
||||
@ConfigurationSettable(
|
||||
configurationClass = CoordinatorStateTableConfig.class,
|
||||
methodName = "deletionProtectionEnabled")
|
||||
private Boolean coordinatorStateDeletionProtectionEnabled;
|
||||
|
||||
@ConfigurationSettable(configurationClass = CoordinatorStateTableConfig.class, methodName = "tags")
|
||||
private TagCollection coordinatorStateTags;
|
||||
}
|
||||
|
|
@ -26,8 +26,7 @@ public class DatePropertyValueDecoder implements IPropertyValueDecoder<Date> {
|
|||
/**
|
||||
* Constructor.
|
||||
*/
|
||||
DatePropertyValueDecoder() {
|
||||
}
|
||||
DatePropertyValueDecoder() {}
|
||||
|
||||
/**
|
||||
* @param value property value as String
|
||||
|
|
@ -49,5 +48,4 @@ public class DatePropertyValueDecoder implements IPropertyValueDecoder<Date> {
|
|||
public List<Class<Date>> getSupportedTypes() {
|
||||
return Arrays.asList(Date.class);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,11 +29,10 @@ import java.util.function.Consumer;
|
|||
import java.util.function.Function;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
import org.apache.commons.beanutils.ConvertUtilsBean;
|
||||
import org.apache.commons.lang3.ClassUtils;
|
||||
|
||||
import com.google.common.collect.HashMultimap;
|
||||
import com.google.common.collect.Multimap;
|
||||
import org.apache.commons.beanutils.ConvertUtilsBean;
|
||||
import org.apache.commons.lang3.ClassUtils;
|
||||
|
||||
class DynaBeanBuilderSupport {
|
||||
|
||||
|
|
@ -48,8 +47,8 @@ class DynaBeanBuilderSupport {
|
|||
private final Multimap<String, TypeTag> properties = HashMultimap.create();
|
||||
private final Map<String, Object> values = new HashMap<>();
|
||||
|
||||
DynaBeanBuilderSupport(Class<?> destinedClass, ConvertUtilsBean convertUtilsBean,
|
||||
List<String> classPrefixSearchList) {
|
||||
DynaBeanBuilderSupport(
|
||||
Class<?> destinedClass, ConvertUtilsBean convertUtilsBean, List<String> classPrefixSearchList) {
|
||||
this.destinedClass = destinedClass;
|
||||
this.convertUtilsBean = convertUtilsBean;
|
||||
this.classPrefixSearchList = classPrefixSearchList;
|
||||
|
|
@ -107,7 +106,8 @@ class DynaBeanBuilderSupport {
|
|||
return new BuilderDynaBean(t.type, convertUtilsBean, null, classPrefixSearchList);
|
||||
}
|
||||
return null;
|
||||
}).orElse(null);
|
||||
})
|
||||
.orElse(null);
|
||||
}
|
||||
|
||||
boolean hasValue(String name) {
|
||||
|
|
@ -157,8 +157,11 @@ class DynaBeanBuilderSupport {
|
|||
|
||||
void set(String name, Object value) {
|
||||
if (value instanceof String && properties.get(name).stream().anyMatch(t -> t.type.isEnum())) {
|
||||
TypeTag typeTag = properties.get(name).stream().filter(t -> t.type.isEnum()).findFirst().orElseThrow(
|
||||
() -> new IllegalStateException("Expected enum type for " + name + ", but couldn't find it."));
|
||||
TypeTag typeTag = properties.get(name).stream()
|
||||
.filter(t -> t.type.isEnum())
|
||||
.findFirst()
|
||||
.orElseThrow(() ->
|
||||
new IllegalStateException("Expected enum type for " + name + ", but couldn't find it."));
|
||||
Class<? extends Enum> enumClass = (Class<? extends Enum>) typeTag.type;
|
||||
values.put(name, Enum.valueOf(enumClass, value.toString()));
|
||||
} else {
|
||||
|
|
@ -174,9 +177,11 @@ class DynaBeanBuilderSupport {
|
|||
private Object getArgument(Map.Entry<String, Object> setValue) {
|
||||
Object argument = setValue.getValue();
|
||||
if (argument instanceof Object[]) {
|
||||
TypeTag arrayType = properties.get(setValue.getKey()).stream().filter(t -> t.type.isArray()).findFirst()
|
||||
.orElseThrow(() -> new IllegalStateException(String
|
||||
.format("Received Object[] for %s but can't find corresponding type", setValue.getKey())));
|
||||
TypeTag arrayType = properties.get(setValue.getKey()).stream()
|
||||
.filter(t -> t.type.isArray())
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new IllegalStateException(String.format(
|
||||
"Received Object[] for %s but can't find corresponding type", setValue.getKey())));
|
||||
Object[] arrayValues = (Object[]) argument;
|
||||
Object[] destination = (Object[]) Array.newInstance(arrayType.type.getComponentType(), arrayValues.length);
|
||||
|
||||
|
|
@ -212,9 +217,11 @@ class DynaBeanBuilderSupport {
|
|||
for (Map.Entry<String, Object> setValue : values.entrySet()) {
|
||||
Object argument = getArgument(setValue);
|
||||
Method mutator = properties.get(setValue.getKey()).stream()
|
||||
.filter(t -> ClassUtils.isAssignable(argument.getClass(), t.type)).findFirst()
|
||||
.map(a -> a.builderMethod).orElseThrow(
|
||||
() -> new IllegalStateException(String.format("Unable to find mutator for %s of type %s",
|
||||
.filter(t -> ClassUtils.isAssignable(argument.getClass(), t.type))
|
||||
.findFirst()
|
||||
.map(a -> a.builderMethod)
|
||||
.orElseThrow(() -> new IllegalStateException(String.format(
|
||||
"Unable to find mutator for %s of type %s",
|
||||
setValue.getKey(), argument.getClass().getName())));
|
||||
try {
|
||||
source = mutator.invoke(source, argument);
|
||||
|
|
@ -236,7 +243,6 @@ class DynaBeanBuilderSupport {
|
|||
} catch (IllegalAccessException | NoSuchMethodException | InvocationTargetException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Collection<String> getPropertyNames() {
|
||||
|
|
@ -249,5 +255,4 @@ class DynaBeanBuilderSupport {
|
|||
}
|
||||
return new ArrayList<>(properties.get(name));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,8 +30,8 @@ class DynaBeanCreateSupport {
|
|||
private final List<TypeTag> createTypes = new ArrayList<>();
|
||||
private Object[] createValues = null;
|
||||
|
||||
DynaBeanCreateSupport(Class<?> destinedClass, ConvertUtilsBean convertUtilsBean,
|
||||
List<String> classPrefixSearchList) {
|
||||
DynaBeanCreateSupport(
|
||||
Class<?> destinedClass, ConvertUtilsBean convertUtilsBean, List<String> classPrefixSearchList) {
|
||||
this.destinedClass = destinedClass;
|
||||
this.convertUtilsBean = convertUtilsBean;
|
||||
this.classPrefixSearchList = classPrefixSearchList;
|
||||
|
|
@ -58,8 +58,8 @@ class DynaBeanCreateSupport {
|
|||
|
||||
Object build() {
|
||||
|
||||
Method createMethod = DynaBeanBuilderUtils.getMethod(destinedClass, "create",
|
||||
createTypes.stream().map(t -> t.type).toArray(i -> new Class<?>[i]));
|
||||
Method createMethod = DynaBeanBuilderUtils.getMethod(
|
||||
destinedClass, "create", createTypes.stream().map(t -> t.type).toArray(i -> new Class<?>[i]));
|
||||
Object arguments[] = new Object[createValues.length];
|
||||
for (int i = 0; i < createValues.length; ++i) {
|
||||
if (createValues[i] instanceof BuilderDynaBean) {
|
||||
|
|
@ -77,8 +77,8 @@ class DynaBeanCreateSupport {
|
|||
return createValues[index];
|
||||
} else {
|
||||
if (createValues[index] == null) {
|
||||
createValues[index] = new BuilderDynaBean(createTypes.get(index).type, convertUtilsBean, null,
|
||||
classPrefixSearchList);
|
||||
createValues[index] = new BuilderDynaBean(
|
||||
createTypes.get(index).type, convertUtilsBean, null, classPrefixSearchList);
|
||||
}
|
||||
return createValues[index];
|
||||
}
|
||||
|
|
@ -89,13 +89,11 @@ class DynaBeanCreateSupport {
|
|||
public void set(String name, int index, Object value) {
|
||||
if (StringUtils.isEmpty(name)) {
|
||||
if (index >= createValues.length) {
|
||||
throw new IllegalArgumentException(
|
||||
String.format("%d exceeds the maximum number of arguments (%d) for %s", index,
|
||||
createValues.length, destinedClass.getName()));
|
||||
throw new IllegalArgumentException(String.format(
|
||||
"%d exceeds the maximum number of arguments (%d) for %s",
|
||||
index, createValues.length, destinedClass.getName()));
|
||||
}
|
||||
createValues[index] = value;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,21 +26,28 @@ public class FanoutConfigBean implements RetrievalConfigBuilder {
|
|||
|
||||
@ConfigurationSettable(configurationClass = FanOutConfig.class)
|
||||
private int maxDescribeStreamSummaryRetries;
|
||||
|
||||
@ConfigurationSettable(configurationClass = FanOutConfig.class)
|
||||
private String consumerArn;
|
||||
|
||||
@ConfigurationSettable(configurationClass = FanOutConfig.class)
|
||||
private String consumerName;
|
||||
|
||||
@ConfigurationSettable(configurationClass = FanOutConfig.class)
|
||||
private int maxDescribeStreamConsumerRetries;
|
||||
|
||||
@ConfigurationSettable(configurationClass = FanOutConfig.class)
|
||||
private int registerStreamConsumerRetries;
|
||||
|
||||
@ConfigurationSettable(configurationClass = FanOutConfig.class)
|
||||
private long retryBackoffMillis;
|
||||
|
||||
@Override
|
||||
public FanOutConfig build(KinesisAsyncClient kinesisAsyncClient, MultiLangDaemonConfiguration parent) {
|
||||
return ConfigurationSettableUtils.resolveFields(this, new FanOutConfig(kinesisAsyncClient).applicationName(parent.getApplicationName())
|
||||
return ConfigurationSettableUtils.resolveFields(
|
||||
this,
|
||||
new FanOutConfig(kinesisAsyncClient)
|
||||
.applicationName(parent.getApplicationName())
|
||||
.streamName(parent.getStreamName()));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,41 @@
|
|||
/*
|
||||
* Copyright 2024 Amazon.com, Inc. or its affiliates.
|
||||
* Licensed under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package software.amazon.kinesis.multilang.config;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import software.amazon.kinesis.leases.LeaseManagementConfig;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
public class GracefulLeaseHandoffConfigBean {
|
||||
|
||||
interface GracefulLeaseHandoffConfigBeanDelegate {
|
||||
Long getGracefulLeaseHandoffTimeoutMillis();
|
||||
|
||||
void setGracefulLeaseHandoffTimeoutMillis(Long value);
|
||||
|
||||
Boolean getIsGracefulLeaseHandoffEnabled();
|
||||
|
||||
void setIsGracefulLeaseHandoffEnabled(Boolean value);
|
||||
}
|
||||
|
||||
@ConfigurationSettable(configurationClass = LeaseManagementConfig.GracefulLeaseHandoffConfig.class)
|
||||
private Long gracefulLeaseHandoffTimeoutMillis;
|
||||
|
||||
@ConfigurationSettable(configurationClass = LeaseManagementConfig.GracefulLeaseHandoffConfig.class)
|
||||
private Boolean isGracefulLeaseHandoffEnabled;
|
||||
}
|
||||
|
|
@ -25,8 +25,7 @@ class IntegerPropertyValueDecoder implements IPropertyValueDecoder<Integer> {
|
|||
/**
|
||||
* Constructor.
|
||||
*/
|
||||
IntegerPropertyValueDecoder() {
|
||||
}
|
||||
IntegerPropertyValueDecoder() {}
|
||||
|
||||
/**
|
||||
* @param value property value as String
|
||||
|
|
|
|||
|
|
@ -19,17 +19,16 @@ import java.io.InputStream;
|
|||
import java.lang.reflect.InvocationTargetException;
|
||||
import java.util.Properties;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.beanutils.BeanUtilsBean;
|
||||
import org.apache.commons.beanutils.ConvertUtilsBean;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.Validate;
|
||||
import software.amazon.awssdk.arns.Arn;
|
||||
import software.amazon.kinesis.common.StreamIdentifier;
|
||||
|
||||
/**
|
||||
* KinesisClientLibConfigurator constructs a KinesisClientLibConfiguration from java properties file. The following
|
||||
* three properties must be provided. 1) "applicationName" 2) "streamName" 3) "AWSCredentialsProvider"
|
||||
* three properties must be provided. 1) "applicationName" 2) "streamName" 3) "AwsCredentialsProvider"
|
||||
* KinesisClientLibConfigurator will help to automatically assign the value of "workerId" if this property is not
|
||||
* provided. In the specified properties file, any properties, which matches the variable name in
|
||||
* KinesisClientLibConfiguration and has a corresponding "with{variableName}" setter method, will be read in, and its
|
||||
|
|
@ -62,7 +61,8 @@ public class KinesisClientLibConfigurator {
|
|||
public MultiLangDaemonConfiguration getConfiguration(Properties properties) {
|
||||
properties.entrySet().forEach(e -> {
|
||||
try {
|
||||
utilsBean.setProperty(configuration, (String) e.getKey(), e.getValue());
|
||||
log.info("Processing (key={}, value={})", e.getKey(), e.getValue());
|
||||
utilsBean.setProperty(configuration, processKey((String) e.getKey()), e.getValue());
|
||||
} catch (IllegalAccessException | InvocationTargetException ex) {
|
||||
throw new RuntimeException(ex);
|
||||
}
|
||||
|
|
@ -70,7 +70,8 @@ public class KinesisClientLibConfigurator {
|
|||
|
||||
Validate.notBlank(configuration.getApplicationName(), "Application name is required");
|
||||
|
||||
if (configuration.getStreamArn() != null && !configuration.getStreamArn().trim().isEmpty()) {
|
||||
if (configuration.getStreamArn() != null
|
||||
&& !configuration.getStreamArn().trim().isEmpty()) {
|
||||
final Arn streamArnObj = Arn.fromString(configuration.getStreamArn());
|
||||
StreamIdentifier.validateArn(streamArnObj);
|
||||
// Parse out the stream Name from the Arn (and/or override existing value for Stream Name)
|
||||
|
|
@ -78,8 +79,12 @@ public class KinesisClientLibConfigurator {
|
|||
configuration.setStreamName(streamNameFromArn);
|
||||
}
|
||||
|
||||
Validate.notBlank(configuration.getStreamName(), "Stream name or Stream Arn is required. Stream Arn takes precedence if both are passed in.");
|
||||
Validate.isTrue(configuration.getKinesisCredentialsProvider().isDirty(), "A basic set of AWS credentials must be provided");
|
||||
Validate.notBlank(
|
||||
configuration.getStreamName(),
|
||||
"Stream name or Stream Arn is required. Stream Arn takes precedence if both are passed in.");
|
||||
Validate.isTrue(
|
||||
configuration.getKinesisCredentialsProvider().isDirty(),
|
||||
"A basic set of AWS credentials must be provided");
|
||||
|
||||
return configuration;
|
||||
}
|
||||
|
|
@ -106,4 +111,16 @@ public class KinesisClientLibConfigurator {
|
|||
return getConfiguration(properties);
|
||||
}
|
||||
|
||||
/**
|
||||
* Processes a configuration key to normalize AWS credentials provider naming. Necessary to conform to
|
||||
* autogenerated setters.
|
||||
* @param key the config param key
|
||||
* @return case-configured param key name
|
||||
*/
|
||||
String processKey(String key) {
|
||||
if (key.toLowerCase().startsWith("awscredentialsprovider")) {
|
||||
key = key.replaceAll("(?i)awscredentialsprovider", "awsCredentialsProvider");
|
||||
}
|
||||
return key;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ package software.amazon.kinesis.multilang.config;
|
|||
|
||||
import java.lang.reflect.InvocationTargetException;
|
||||
import java.net.URI;
|
||||
import java.time.Duration;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.Date;
|
||||
|
|
@ -27,21 +28,21 @@ import java.util.Set;
|
|||
import java.util.UUID;
|
||||
import java.util.function.Function;
|
||||
|
||||
import org.apache.commons.beanutils.BeanUtilsBean;
|
||||
import org.apache.commons.beanutils.ConvertUtilsBean;
|
||||
import org.apache.commons.beanutils.Converter;
|
||||
import org.apache.commons.beanutils.converters.ArrayConverter;
|
||||
import org.apache.commons.beanutils.converters.StringConverter;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import lombok.experimental.Delegate;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.beanutils.BeanUtilsBean;
|
||||
import org.apache.commons.beanutils.ConvertUtilsBean;
|
||||
import org.apache.commons.beanutils.Converter;
|
||||
import org.apache.commons.beanutils.converters.ArrayConverter;
|
||||
import org.apache.commons.beanutils.converters.StringConverter;
|
||||
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
|
||||
import software.amazon.awssdk.regions.Region;
|
||||
import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient;
|
||||
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
|
||||
import software.amazon.awssdk.services.dynamodb.model.BillingMode;
|
||||
import software.amazon.awssdk.services.kinesis.KinesisAsyncClient;
|
||||
import software.amazon.awssdk.services.kinesis.KinesisAsyncClientBuilder;
|
||||
import software.amazon.kinesis.checkpoint.CheckpointConfig;
|
||||
|
|
@ -56,7 +57,9 @@ import software.amazon.kinesis.leases.ShardPrioritization;
|
|||
import software.amazon.kinesis.lifecycle.LifecycleConfig;
|
||||
import software.amazon.kinesis.metrics.MetricsConfig;
|
||||
import software.amazon.kinesis.metrics.MetricsLevel;
|
||||
import software.amazon.kinesis.multilang.config.credentials.V2CredentialWrapper;
|
||||
import software.amazon.kinesis.multilang.config.converter.DurationConverter;
|
||||
import software.amazon.kinesis.multilang.config.converter.TagConverter;
|
||||
import software.amazon.kinesis.multilang.config.converter.TagConverter.TagCollection;
|
||||
import software.amazon.kinesis.processor.ProcessorConfig;
|
||||
import software.amazon.kinesis.processor.ShardRecordProcessorFactory;
|
||||
import software.amazon.kinesis.retrieval.RetrievalConfig;
|
||||
|
|
@ -74,7 +77,6 @@ public class MultiLangDaemonConfiguration {
|
|||
private String streamName;
|
||||
private String streamArn;
|
||||
|
||||
|
||||
@ConfigurationSettable(configurationClass = ConfigsBuilder.class)
|
||||
private String tableName;
|
||||
|
||||
|
|
@ -86,20 +88,37 @@ public class MultiLangDaemonConfiguration {
|
|||
|
||||
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
|
||||
private long failoverTimeMillis;
|
||||
|
||||
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
|
||||
private Boolean enablePriorityLeaseAssignment;
|
||||
|
||||
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
|
||||
private Boolean leaseTableDeletionProtectionEnabled;
|
||||
|
||||
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
|
||||
private Boolean leaseTablePitrEnabled;
|
||||
|
||||
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
|
||||
private long shardSyncIntervalMillis;
|
||||
|
||||
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
|
||||
private boolean cleanupLeasesUponShardCompletion;
|
||||
|
||||
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
|
||||
private boolean ignoreUnexpectedChildShards;
|
||||
|
||||
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
|
||||
private int maxLeasesForWorker;
|
||||
|
||||
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
|
||||
private int maxLeasesToStealAtOneTime;
|
||||
|
||||
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
|
||||
private int initialLeaseTableReadCapacity;
|
||||
|
||||
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
|
||||
private int initialLeaseTableWriteCapacity;
|
||||
|
||||
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class, methodName = "initialPositionInStream")
|
||||
@ConfigurationSettable(configurationClass = RetrievalConfig.class)
|
||||
private InitialPositionInStreamExtended initialPositionInStreamExtended;
|
||||
|
|
@ -112,14 +131,16 @@ public class MultiLangDaemonConfiguration {
|
|||
}
|
||||
|
||||
public void setInitialPositionInStream(InitialPositionInStream initialPositionInStream) {
|
||||
this.initialPositionInStreamExtended = InitialPositionInStreamExtended
|
||||
.newInitialPosition(initialPositionInStream);
|
||||
this.initialPositionInStreamExtended =
|
||||
InitialPositionInStreamExtended.newInitialPosition(initialPositionInStream);
|
||||
}
|
||||
|
||||
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
|
||||
private int maxLeaseRenewalThreads;
|
||||
|
||||
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
|
||||
private long listShardsBackoffTimeInMillis;
|
||||
|
||||
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
|
||||
private int maxListShardsRetryAttempts;
|
||||
|
||||
|
|
@ -129,24 +150,34 @@ public class MultiLangDaemonConfiguration {
|
|||
|
||||
@ConfigurationSettable(configurationClass = CoordinatorConfig.class)
|
||||
private long parentShardPollIntervalMillis;
|
||||
|
||||
@ConfigurationSettable(configurationClass = CoordinatorConfig.class)
|
||||
private ShardPrioritization shardPrioritization;
|
||||
|
||||
@ConfigurationSettable(configurationClass = CoordinatorConfig.class)
|
||||
private boolean skipShardSyncAtWorkerInitializationIfLeasesExist;
|
||||
|
||||
@ConfigurationSettable(configurationClass = CoordinatorConfig.class)
|
||||
private long schedulerInitializationBackoffTimeMillis;
|
||||
|
||||
@ConfigurationSettable(configurationClass = CoordinatorConfig.class)
|
||||
private CoordinatorConfig.ClientVersionConfig clientVersionConfig;
|
||||
|
||||
@ConfigurationSettable(configurationClass = LifecycleConfig.class)
|
||||
private long taskBackoffTimeMillis;
|
||||
|
||||
@ConfigurationSettable(configurationClass = MetricsConfig.class)
|
||||
private long metricsBufferTimeMillis;
|
||||
|
||||
@ConfigurationSettable(configurationClass = MetricsConfig.class)
|
||||
private int metricsMaxQueueSize;
|
||||
|
||||
@ConfigurationSettable(configurationClass = MetricsConfig.class)
|
||||
private MetricsLevel metricsLevel;
|
||||
|
||||
@ConfigurationSettable(configurationClass = LifecycleConfig.class, convertToOptional = true)
|
||||
private Long logWarningForTaskAfterMillis;
|
||||
|
||||
@ConfigurationSettable(configurationClass = MetricsConfig.class)
|
||||
private Set<String> metricsEnabledDimensions;
|
||||
|
||||
|
|
@ -161,9 +192,26 @@ public class MultiLangDaemonConfiguration {
|
|||
private RetrievalMode retrievalMode = RetrievalMode.DEFAULT;
|
||||
|
||||
private final FanoutConfigBean fanoutConfig = new FanoutConfigBean();
|
||||
|
||||
@Delegate(types = PollingConfigBean.PollingConfigBeanDelegate.class)
|
||||
private final PollingConfigBean pollingConfig = new PollingConfigBean();
|
||||
|
||||
@Delegate(types = GracefulLeaseHandoffConfigBean.GracefulLeaseHandoffConfigBeanDelegate.class)
|
||||
private final GracefulLeaseHandoffConfigBean gracefulLeaseHandoffConfigBean = new GracefulLeaseHandoffConfigBean();
|
||||
|
||||
@Delegate(
|
||||
types = WorkerUtilizationAwareAssignmentConfigBean.WorkerUtilizationAwareAssignmentConfigBeanDelegate.class)
|
||||
private final WorkerUtilizationAwareAssignmentConfigBean workerUtilizationAwareAssignmentConfigBean =
|
||||
new WorkerUtilizationAwareAssignmentConfigBean();
|
||||
|
||||
@Delegate(types = WorkerMetricStatsTableConfigBean.WorkerMetricsTableConfigBeanDelegate.class)
|
||||
private final WorkerMetricStatsTableConfigBean workerMetricStatsTableConfigBean =
|
||||
new WorkerMetricStatsTableConfigBean();
|
||||
|
||||
@Delegate(types = CoordinatorStateTableConfigBean.CoordinatorStateConfigBeanDelegate.class)
|
||||
private final CoordinatorStateTableConfigBean coordinatorStateTableConfigBean =
|
||||
new CoordinatorStateTableConfigBean();
|
||||
|
||||
private boolean validateSequenceNumberBeforeCheckpointing;
|
||||
|
||||
private long shutdownGraceMillis;
|
||||
|
|
@ -171,19 +219,19 @@ public class MultiLangDaemonConfiguration {
|
|||
|
||||
private final BuilderDynaBean kinesisCredentialsProvider;
|
||||
|
||||
public void setAWSCredentialsProvider(String providerString) {
|
||||
public void setAwsCredentialsProvider(String providerString) {
|
||||
kinesisCredentialsProvider.set("", providerString);
|
||||
}
|
||||
|
||||
private final BuilderDynaBean dynamoDBCredentialsProvider;
|
||||
|
||||
public void setAWSCredentialsProviderDynamoDB(String providerString) {
|
||||
public void setAwsCredentialsProviderDynamoDB(String providerString) {
|
||||
dynamoDBCredentialsProvider.set("", providerString);
|
||||
}
|
||||
|
||||
private final BuilderDynaBean cloudWatchCredentialsProvider;
|
||||
|
||||
public void setAWSCredentialsProviderCloudWatch(String providerString) {
|
||||
public void setAwsCredentialsProviderCloudWatch(String providerString) {
|
||||
cloudWatchCredentialsProvider.set("", providerString);
|
||||
}
|
||||
|
||||
|
|
@ -198,61 +246,97 @@ public class MultiLangDaemonConfiguration {
|
|||
this.utilsBean = utilsBean;
|
||||
this.convertUtilsBean = convertUtilsBean;
|
||||
|
||||
convertUtilsBean.register(new Converter() {
|
||||
convertUtilsBean.register(
|
||||
new Converter() {
|
||||
@Override
|
||||
public <T> T convert(Class<T> type, Object value) {
|
||||
Date date = new Date(Long.parseLong(value.toString()) * 1000L);
|
||||
return type.cast(InitialPositionInStreamExtended.newInitialPositionAtTimestamp(date));
|
||||
}
|
||||
}, InitialPositionInStreamExtended.class);
|
||||
},
|
||||
InitialPositionInStreamExtended.class);
|
||||
|
||||
convertUtilsBean.register(new Converter() {
|
||||
convertUtilsBean.register(
|
||||
new Converter() {
|
||||
@Override
|
||||
public <T> T convert(Class<T> type, Object value) {
|
||||
return type.cast(MetricsLevel.valueOf(value.toString().toUpperCase()));
|
||||
}
|
||||
}, MetricsLevel.class);
|
||||
},
|
||||
MetricsLevel.class);
|
||||
|
||||
convertUtilsBean.register(new Converter() {
|
||||
convertUtilsBean.register(
|
||||
new Converter() {
|
||||
@Override
|
||||
public <T> T convert(Class<T> type, Object value) {
|
||||
return type.cast(InitialPositionInStream.valueOf(value.toString().toUpperCase()));
|
||||
return type.cast(
|
||||
InitialPositionInStream.valueOf(value.toString().toUpperCase()));
|
||||
}
|
||||
}, InitialPositionInStream.class);
|
||||
},
|
||||
InitialPositionInStream.class);
|
||||
|
||||
convertUtilsBean.register(new Converter() {
|
||||
convertUtilsBean.register(
|
||||
new Converter() {
|
||||
@Override
|
||||
public <T> T convert(Class<T> type, Object value) {
|
||||
return type.cast(CoordinatorConfig.ClientVersionConfig.valueOf(
|
||||
value.toString().toUpperCase()));
|
||||
}
|
||||
},
|
||||
CoordinatorConfig.ClientVersionConfig.class);
|
||||
|
||||
convertUtilsBean.register(
|
||||
new Converter() {
|
||||
@Override
|
||||
public <T> T convert(Class<T> type, Object value) {
|
||||
return type.cast(BillingMode.valueOf(value.toString().toUpperCase()));
|
||||
}
|
||||
},
|
||||
BillingMode.class);
|
||||
|
||||
convertUtilsBean.register(
|
||||
new Converter() {
|
||||
@Override
|
||||
public <T> T convert(Class<T> type, Object value) {
|
||||
return type.cast(URI.create(value.toString()));
|
||||
}
|
||||
}, URI.class);
|
||||
},
|
||||
URI.class);
|
||||
|
||||
convertUtilsBean.register(new Converter() {
|
||||
convertUtilsBean.register(
|
||||
new Converter() {
|
||||
@Override
|
||||
public <T> T convert(Class<T> type, Object value) {
|
||||
return type.cast(RetrievalMode.from(value.toString()));
|
||||
}
|
||||
}, RetrievalMode.class);
|
||||
},
|
||||
RetrievalMode.class);
|
||||
|
||||
convertUtilsBean.register(new Converter() {
|
||||
convertUtilsBean.register(
|
||||
new Converter() {
|
||||
@Override
|
||||
public <T> T convert(final Class<T> type, final Object value) {
|
||||
return type.cast(Region.of(value.toString()));
|
||||
}
|
||||
}, Region.class);
|
||||
},
|
||||
Region.class);
|
||||
|
||||
convertUtilsBean.register(new DurationConverter(), Duration.class);
|
||||
convertUtilsBean.register(new TagConverter(), TagCollection.class);
|
||||
|
||||
ArrayConverter arrayConverter = new ArrayConverter(String[].class, new StringConverter());
|
||||
arrayConverter.setDelimiter(',');
|
||||
convertUtilsBean.register(arrayConverter, String[].class);
|
||||
AWSCredentialsProviderPropertyValueDecoder oldCredentialsDecoder = new AWSCredentialsProviderPropertyValueDecoder();
|
||||
Function<String, ?> converter = s -> new V2CredentialWrapper(oldCredentialsDecoder.decodeValue(s));
|
||||
AwsCredentialsProviderPropertyValueDecoder credentialsDecoder =
|
||||
new AwsCredentialsProviderPropertyValueDecoder();
|
||||
Function<String, ?> converter = credentialsDecoder::decodeValue;
|
||||
|
||||
this.kinesisCredentialsProvider = new BuilderDynaBean(AwsCredentialsProvider.class, convertUtilsBean,
|
||||
converter, CREDENTIALS_DEFAULT_SEARCH_PATH);
|
||||
this.dynamoDBCredentialsProvider = new BuilderDynaBean(AwsCredentialsProvider.class, convertUtilsBean,
|
||||
converter, CREDENTIALS_DEFAULT_SEARCH_PATH);
|
||||
this.cloudWatchCredentialsProvider = new BuilderDynaBean(AwsCredentialsProvider.class, convertUtilsBean,
|
||||
converter, CREDENTIALS_DEFAULT_SEARCH_PATH);
|
||||
this.kinesisCredentialsProvider = new BuilderDynaBean(
|
||||
AwsCredentialsProvider.class, convertUtilsBean, converter, CREDENTIALS_DEFAULT_SEARCH_PATH);
|
||||
this.dynamoDBCredentialsProvider = new BuilderDynaBean(
|
||||
AwsCredentialsProvider.class, convertUtilsBean, converter, CREDENTIALS_DEFAULT_SEARCH_PATH);
|
||||
this.cloudWatchCredentialsProvider = new BuilderDynaBean(
|
||||
AwsCredentialsProvider.class, convertUtilsBean, converter, CREDENTIALS_DEFAULT_SEARCH_PATH);
|
||||
|
||||
this.kinesisClient = new BuilderDynaBean(KinesisAsyncClient.class, convertUtilsBean);
|
||||
this.dynamoDbClient = new BuilderDynaBean(DynamoDbAsyncClient.class, convertUtilsBean);
|
||||
|
|
@ -298,8 +382,8 @@ public class MultiLangDaemonConfiguration {
|
|||
return credsBuilder.build(AwsCredentialsProvider.class);
|
||||
}
|
||||
|
||||
private void updateCredentials(BuilderDynaBean toUpdate, AwsCredentialsProvider primary,
|
||||
AwsCredentialsProvider secondary) {
|
||||
private void updateCredentials(
|
||||
BuilderDynaBean toUpdate, AwsCredentialsProvider primary, AwsCredentialsProvider secondary) {
|
||||
|
||||
if (toUpdate.hasValue("credentialsProvider")) {
|
||||
return;
|
||||
|
|
@ -327,8 +411,24 @@ public class MultiLangDaemonConfiguration {
|
|||
}
|
||||
|
||||
private void handleRetrievalConfig(RetrievalConfig retrievalConfig, ConfigsBuilder configsBuilder) {
|
||||
retrievalConfig
|
||||
.retrievalSpecificConfig(retrievalMode.builder(this).build(configsBuilder.kinesisClient(), this));
|
||||
retrievalConfig.retrievalSpecificConfig(
|
||||
retrievalMode.builder(this).build(configsBuilder.kinesisClient(), this));
|
||||
}
|
||||
|
||||
private void handleCoordinatorConfig(CoordinatorConfig coordinatorConfig) {
|
||||
ConfigurationSettableUtils.resolveFields(
|
||||
this.coordinatorStateTableConfigBean, coordinatorConfig.coordinatorStateTableConfig());
|
||||
}
|
||||
|
||||
private void handleLeaseManagementConfig(LeaseManagementConfig leaseManagementConfig) {
|
||||
ConfigurationSettableUtils.resolveFields(
|
||||
this.gracefulLeaseHandoffConfigBean, leaseManagementConfig.gracefulLeaseHandoffConfig());
|
||||
ConfigurationSettableUtils.resolveFields(
|
||||
this.workerUtilizationAwareAssignmentConfigBean,
|
||||
leaseManagementConfig.workerUtilizationAwareAssignmentConfig());
|
||||
ConfigurationSettableUtils.resolveFields(
|
||||
this.workerMetricStatsTableConfigBean,
|
||||
leaseManagementConfig.workerUtilizationAwareAssignmentConfig().workerMetricsTableConfig());
|
||||
}
|
||||
|
||||
private Object adjustKinesisHttpConfiguration(Object builderObj) {
|
||||
|
|
@ -351,8 +451,14 @@ public class MultiLangDaemonConfiguration {
|
|||
final RetrievalConfig retrievalConfig;
|
||||
|
||||
public Scheduler build() {
|
||||
return new Scheduler(checkpointConfig, coordinatorConfig, leaseManagementConfig, lifecycleConfig,
|
||||
metricsConfig, processorConfig, retrievalConfig);
|
||||
return new Scheduler(
|
||||
checkpointConfig,
|
||||
coordinatorConfig,
|
||||
leaseManagementConfig,
|
||||
lifecycleConfig,
|
||||
metricsConfig,
|
||||
processorConfig,
|
||||
retrievalConfig);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -365,19 +471,25 @@ public class MultiLangDaemonConfiguration {
|
|||
updateCredentials(dynamoDbClient, dynamoDbCreds, kinesisCreds);
|
||||
updateCredentials(cloudWatchClient, cloudwatchCreds, kinesisCreds);
|
||||
|
||||
KinesisAsyncClient kinesisAsyncClient = kinesisClient.build(KinesisAsyncClient.class,
|
||||
this::adjustKinesisHttpConfiguration);
|
||||
KinesisAsyncClient kinesisAsyncClient =
|
||||
kinesisClient.build(KinesisAsyncClient.class, this::adjustKinesisHttpConfiguration);
|
||||
DynamoDbAsyncClient dynamoDbAsyncClient = dynamoDbClient.build(DynamoDbAsyncClient.class);
|
||||
CloudWatchAsyncClient cloudWatchAsyncClient = cloudWatchClient.build(CloudWatchAsyncClient.class);
|
||||
|
||||
ConfigsBuilder configsBuilder = new ConfigsBuilder(streamName, applicationName, kinesisAsyncClient,
|
||||
dynamoDbAsyncClient, cloudWatchAsyncClient, workerIdentifier, shardRecordProcessorFactory);
|
||||
ConfigsBuilder configsBuilder = new ConfigsBuilder(
|
||||
streamName,
|
||||
applicationName,
|
||||
kinesisAsyncClient,
|
||||
dynamoDbAsyncClient,
|
||||
cloudWatchAsyncClient,
|
||||
workerIdentifier,
|
||||
shardRecordProcessorFactory);
|
||||
|
||||
Map<Class<?>, Object> configObjects = new HashMap<>();
|
||||
addConfigObjects(configObjects, configsBuilder);
|
||||
|
||||
resolveFields(configObjects, Collections.singleton(ConfigsBuilder.class),
|
||||
Collections.singleton(PollingConfig.class));
|
||||
resolveFields(
|
||||
configObjects, Collections.singleton(ConfigsBuilder.class), Collections.singleton(PollingConfig.class));
|
||||
|
||||
CoordinatorConfig coordinatorConfig = configsBuilder.coordinatorConfig();
|
||||
CheckpointConfig checkpointConfig = configsBuilder.checkpointConfig();
|
||||
|
|
@ -387,19 +499,33 @@ public class MultiLangDaemonConfiguration {
|
|||
ProcessorConfig processorConfig = configsBuilder.processorConfig();
|
||||
RetrievalConfig retrievalConfig = configsBuilder.retrievalConfig();
|
||||
|
||||
addConfigObjects(configObjects, coordinatorConfig, checkpointConfig, leaseManagementConfig, lifecycleConfig,
|
||||
metricsConfig, processorConfig, retrievalConfig);
|
||||
addConfigObjects(
|
||||
configObjects,
|
||||
coordinatorConfig,
|
||||
checkpointConfig,
|
||||
leaseManagementConfig,
|
||||
lifecycleConfig,
|
||||
metricsConfig,
|
||||
processorConfig,
|
||||
retrievalConfig);
|
||||
|
||||
handleCoordinatorConfig(coordinatorConfig);
|
||||
handleLeaseManagementConfig(leaseManagementConfig);
|
||||
handleRetrievalConfig(retrievalConfig, configsBuilder);
|
||||
|
||||
resolveFields(configObjects, null, new HashSet<>(Arrays.asList(ConfigsBuilder.class, PollingConfig.class)));
|
||||
|
||||
return new ResolvedConfiguration(coordinatorConfig, checkpointConfig, leaseManagementConfig, lifecycleConfig,
|
||||
metricsConfig, processorConfig, retrievalConfig);
|
||||
return new ResolvedConfiguration(
|
||||
coordinatorConfig,
|
||||
checkpointConfig,
|
||||
leaseManagementConfig,
|
||||
lifecycleConfig,
|
||||
metricsConfig,
|
||||
processorConfig,
|
||||
retrievalConfig);
|
||||
}
|
||||
|
||||
public Scheduler build(ShardRecordProcessorFactory shardRecordProcessorFactory) {
|
||||
return resolvedConfiguration(shardRecordProcessorFactory).build();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -30,34 +30,44 @@ public class PollingConfigBean implements RetrievalConfigBuilder {
|
|||
interface PollingConfigBeanDelegate {
|
||||
|
||||
Integer getRetryGetRecordsInSeconds();
|
||||
|
||||
void setRetryGetRecordsInSeconds(Integer value);
|
||||
|
||||
Integer getMaxGetRecordsThreadPool();
|
||||
|
||||
void setMaxGetRecordsThreadPool(Integer value);
|
||||
|
||||
long getIdleTimeBetweenReadsInMillis();
|
||||
|
||||
void setIdleTimeBetweenReadsInMillis(long value);
|
||||
|
||||
int getMaxRecords();
|
||||
|
||||
void setMaxRecords(int value);
|
||||
}
|
||||
|
||||
@ConfigurationSettable(configurationClass = PollingConfig.class, convertToOptional = true)
|
||||
private Integer retryGetRecordsInSeconds;
|
||||
|
||||
@ConfigurationSettable(configurationClass = PollingConfig.class, convertToOptional = true)
|
||||
private Integer maxGetRecordsThreadPool;
|
||||
|
||||
@ConfigurationSettable(configurationClass = PollingConfig.class)
|
||||
private long idleTimeBetweenReadsInMillis;
|
||||
|
||||
@ConfigurationSettable(configurationClass = PollingConfig.class)
|
||||
private int maxRecords;
|
||||
|
||||
public boolean anyPropertiesSet() {
|
||||
return retryGetRecordsInSeconds != null || maxGetRecordsThreadPool != null || idleTimeBetweenReadsInMillis != 0 || maxRecords != 0;
|
||||
return retryGetRecordsInSeconds != null
|
||||
|| maxGetRecordsThreadPool != null
|
||||
|| idleTimeBetweenReadsInMillis != 0
|
||||
|| maxRecords != 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public PollingConfig build(KinesisAsyncClient kinesisAsyncClient, MultiLangDaemonConfiguration parent) {
|
||||
return ConfigurationSettableUtils.resolveFields(this, new PollingConfig(parent.getStreamName(), kinesisAsyncClient));
|
||||
return ConfigurationSettableUtils.resolveFields(
|
||||
this, new PollingConfig(parent.getStreamName(), kinesisAsyncClient));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,14 +19,14 @@ import java.util.Arrays;
|
|||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.lang3.Validate;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.Validate;
|
||||
|
||||
@Slf4j
|
||||
public enum RetrievalMode {
|
||||
FANOUT(MultiLangDaemonConfiguration::getFanoutConfig), POLLING(
|
||||
MultiLangDaemonConfiguration::getPollingConfig), DEFAULT(RetrievalMode::decideForDefault);
|
||||
FANOUT(MultiLangDaemonConfiguration::getFanoutConfig),
|
||||
POLLING(MultiLangDaemonConfiguration::getPollingConfig),
|
||||
DEFAULT(RetrievalMode::decideForDefault);
|
||||
|
||||
private final Function<MultiLangDaemonConfiguration, RetrievalConfigBuilder> builderFor;
|
||||
|
||||
|
|
|
|||
|
|
@ -15,14 +15,13 @@
|
|||
|
||||
package software.amazon.kinesis.multilang.config;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
class TypeTag {
|
||||
final Class<?> type;
|
||||
final boolean hasConverter;
|
||||
final Method builderMethod;
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,82 @@
|
|||
/*
|
||||
* Copyright 2024 Amazon.com, Inc. or its affiliates.
|
||||
* Licensed under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package software.amazon.kinesis.multilang.config;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import software.amazon.awssdk.services.dynamodb.model.BillingMode;
|
||||
import software.amazon.kinesis.leases.LeaseManagementConfig.WorkerMetricsTableConfig;
|
||||
import software.amazon.kinesis.multilang.config.converter.TagConverter.TagCollection;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
public class WorkerMetricStatsTableConfigBean {
|
||||
|
||||
interface WorkerMetricsTableConfigBeanDelegate {
|
||||
String getWorkerMetricsTableName();
|
||||
|
||||
void setWorkerMetricsTableName(String value);
|
||||
|
||||
BillingMode getWorkerMetricsBillingMode();
|
||||
|
||||
void setWorkerMetricsBillingMode(BillingMode value);
|
||||
|
||||
long getWorkerMetricsReadCapacity();
|
||||
|
||||
void setWorkerMetricsReadCapacity(long value);
|
||||
|
||||
long getWorkerMetricsWriteCapacity();
|
||||
|
||||
void setWorkerMetricsWriteCapacity(long value);
|
||||
|
||||
Boolean getWorkerMetricsPointInTimeRecoveryEnabled();
|
||||
|
||||
void setWorkerMetricsPointInTimeRecoveryEnabled(Boolean value);
|
||||
|
||||
Boolean getWorkerMetricsDeletionProtectionEnabled();
|
||||
|
||||
void setWorkerMetricsDeletionProtectionEnabled(Boolean value);
|
||||
|
||||
TagCollection getWorkerMetricsTags();
|
||||
|
||||
void setWorkerMetricsTags(TagCollection value);
|
||||
}
|
||||
|
||||
@ConfigurationSettable(configurationClass = WorkerMetricsTableConfig.class, methodName = "tableName")
|
||||
private String workerMetricsTableName;
|
||||
|
||||
@ConfigurationSettable(configurationClass = WorkerMetricsTableConfig.class, methodName = "billingMode")
|
||||
private BillingMode workerMetricsBillingMode;
|
||||
|
||||
@ConfigurationSettable(configurationClass = WorkerMetricsTableConfig.class, methodName = "readCapacity")
|
||||
private long workerMetricsReadCapacity;
|
||||
|
||||
@ConfigurationSettable(configurationClass = WorkerMetricsTableConfig.class, methodName = "writeCapacity")
|
||||
private long workerMetricsWriteCapacity;
|
||||
|
||||
@ConfigurationSettable(
|
||||
configurationClass = WorkerMetricsTableConfig.class,
|
||||
methodName = "pointInTimeRecoveryEnabled")
|
||||
private Boolean workerMetricsPointInTimeRecoveryEnabled;
|
||||
|
||||
@ConfigurationSettable(
|
||||
configurationClass = WorkerMetricsTableConfig.class,
|
||||
methodName = "deletionProtectionEnabled")
|
||||
private Boolean workerMetricsDeletionProtectionEnabled;
|
||||
|
||||
@ConfigurationSettable(configurationClass = WorkerMetricsTableConfig.class, methodName = "tags")
|
||||
private TagCollection workerMetricsTags;
|
||||
}
|
||||
|
|
@ -0,0 +1,106 @@
|
|||
/*
|
||||
* Copyright 2024 Amazon.com, Inc. or its affiliates.
|
||||
* Licensed under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package software.amazon.kinesis.multilang.config;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import software.amazon.kinesis.leases.LeaseManagementConfig.WorkerUtilizationAwareAssignmentConfig;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
public class WorkerUtilizationAwareAssignmentConfigBean {
|
||||
|
||||
interface WorkerUtilizationAwareAssignmentConfigBeanDelegate {
|
||||
long getInMemoryWorkerMetricsCaptureFrequencyMillis();
|
||||
|
||||
void setInMemoryWorkerMetricsCaptureFrequencyMillis(long value);
|
||||
|
||||
long getWorkerMetricsReporterFreqInMillis();
|
||||
|
||||
void setWorkerMetricsReporterFreqInMillis(long value);
|
||||
|
||||
int getNoOfPersistedMetricsPerWorkerMetrics();
|
||||
|
||||
void setNoOfPersistedMetricsPerWorkerMetrics(int value);
|
||||
|
||||
Boolean getDisableWorkerMetrics();
|
||||
|
||||
void setDisableWorkerMetrics(Boolean value);
|
||||
|
||||
double getMaxThroughputPerHostKBps();
|
||||
|
||||
void setMaxThroughputPerHostKBps(double value);
|
||||
|
||||
int getDampeningPercentage();
|
||||
|
||||
void setDampeningPercentage(int value);
|
||||
|
||||
int getReBalanceThresholdPercentage();
|
||||
|
||||
void setReBalanceThresholdPercentage(int value);
|
||||
|
||||
Boolean getAllowThroughputOvershoot();
|
||||
|
||||
void setAllowThroughputOvershoot(Boolean value);
|
||||
|
||||
int getVarianceBalancingFrequency();
|
||||
|
||||
void setVarianceBalancingFrequency(int value);
|
||||
|
||||
double getWorkerMetricsEMAAlpha();
|
||||
|
||||
void setWorkerMetricsEMAAlpha(double value);
|
||||
|
||||
void setStaleWorkerMetricsEntryCleanupDuration(Duration value);
|
||||
|
||||
Duration getStaleWorkerMetricsEntryCleanupDuration();
|
||||
}
|
||||
|
||||
@ConfigurationSettable(configurationClass = WorkerUtilizationAwareAssignmentConfig.class)
|
||||
private long inMemoryWorkerMetricsCaptureFrequencyMillis;
|
||||
|
||||
@ConfigurationSettable(configurationClass = WorkerUtilizationAwareAssignmentConfig.class)
|
||||
private long workerMetricsReporterFreqInMillis;
|
||||
|
||||
@ConfigurationSettable(configurationClass = WorkerUtilizationAwareAssignmentConfig.class)
|
||||
private int noOfPersistedMetricsPerWorkerMetrics;
|
||||
|
||||
@ConfigurationSettable(configurationClass = WorkerUtilizationAwareAssignmentConfig.class)
|
||||
private Boolean disableWorkerMetrics;
|
||||
|
||||
@ConfigurationSettable(configurationClass = WorkerUtilizationAwareAssignmentConfig.class)
|
||||
private double maxThroughputPerHostKBps;
|
||||
|
||||
@ConfigurationSettable(configurationClass = WorkerUtilizationAwareAssignmentConfig.class)
|
||||
private int dampeningPercentage;
|
||||
|
||||
@ConfigurationSettable(configurationClass = WorkerUtilizationAwareAssignmentConfig.class)
|
||||
private int reBalanceThresholdPercentage;
|
||||
|
||||
@ConfigurationSettable(configurationClass = WorkerUtilizationAwareAssignmentConfig.class)
|
||||
private Boolean allowThroughputOvershoot;
|
||||
|
||||
@ConfigurationSettable(configurationClass = WorkerUtilizationAwareAssignmentConfig.class)
|
||||
private int varianceBalancingFrequency;
|
||||
|
||||
@ConfigurationSettable(configurationClass = WorkerUtilizationAwareAssignmentConfig.class)
|
||||
private double workerMetricsEMAAlpha;
|
||||
|
||||
@ConfigurationSettable(configurationClass = WorkerUtilizationAwareAssignmentConfig.class)
|
||||
private Duration staleWorkerMetricsEntryCleanupDuration;
|
||||
}
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
/*
|
||||
* Copyright 2024 Amazon.com, Inc. or its affiliates.
|
||||
* Licensed under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package software.amazon.kinesis.multilang.config.converter;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
import org.apache.commons.beanutils.Converter;
|
||||
|
||||
/**
|
||||
* Converter that converts Duration text representation to a Duration object.
|
||||
* Refer to {@code Duration.parse} javadocs for the exact text representation.
|
||||
*/
|
||||
public class DurationConverter implements Converter {
|
||||
|
||||
@Override
|
||||
public <T> T convert(Class<T> type, Object value) {
|
||||
if (value == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (type != Duration.class) {
|
||||
throw new ConversionException("Can only convert to Duration");
|
||||
}
|
||||
|
||||
String durationString = value.toString().trim();
|
||||
final Duration duration = Duration.parse(durationString);
|
||||
if (duration.isNegative()) {
|
||||
throw new ConversionException("Negative values are not permitted for duration: " + durationString);
|
||||
}
|
||||
|
||||
return type.cast(duration);
|
||||
}
|
||||
|
||||
public static class ConversionException extends RuntimeException {
|
||||
public ConversionException(String message) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
/*
|
||||
* Copyright 2024 Amazon.com, Inc. or its affiliates.
|
||||
* Licensed under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package software.amazon.kinesis.multilang.config.converter;
|
||||
|
||||
import java.util.ArrayList;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.beanutils.Converter;
|
||||
import software.amazon.awssdk.services.dynamodb.model.Tag;
|
||||
|
||||
/**
|
||||
* Converter that converts to a Collection of Tag object.
|
||||
* The text format accepted are as follows:
|
||||
* tagPropertyName = key1=value1,key2=value2,...
|
||||
*/
|
||||
@Slf4j
|
||||
public class TagConverter implements Converter {
|
||||
|
||||
@Override
|
||||
public <T> T convert(Class<T> type, Object value) {
|
||||
if (value == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!type.isAssignableFrom(TagCollection.class)) {
|
||||
throw new ConversionException("Can only convert to Collection<Tag>");
|
||||
}
|
||||
|
||||
final TagCollection collection = new TagCollection();
|
||||
final String tagString = value.toString().trim();
|
||||
final String[] keyValuePairs = tagString.split(",");
|
||||
for (String keyValuePair : keyValuePairs) {
|
||||
final String[] tokens = keyValuePair.trim().split("=");
|
||||
if (tokens.length != 2) {
|
||||
log.warn("Invalid tag {}, ignoring it", keyValuePair);
|
||||
continue;
|
||||
}
|
||||
final Tag tag =
|
||||
Tag.builder().key(tokens[0].trim()).value(tokens[1].trim()).build();
|
||||
log.info("Created tag {}", tag);
|
||||
collection.add(tag);
|
||||
}
|
||||
|
||||
return type.cast(collection);
|
||||
}
|
||||
|
||||
public static class ConversionException extends RuntimeException {
|
||||
public ConversionException(String message) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
public static class TagCollection extends ArrayList<Tag> {}
|
||||
}
|
||||
|
|
@ -1,50 +0,0 @@
|
|||
/*
|
||||
* Copyright 2019 Amazon.com, Inc. or its affiliates.
|
||||
* Licensed under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package software.amazon.kinesis.multilang.config.credentials;
|
||||
|
||||
import com.amazonaws.auth.AWSCredentials;
|
||||
import com.amazonaws.auth.AWSCredentialsProvider;
|
||||
import com.amazonaws.auth.AWSSessionCredentials;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import software.amazon.awssdk.auth.credentials.AwsCredentials;
|
||||
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
|
||||
import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
|
||||
|
||||
@RequiredArgsConstructor
|
||||
public class V2CredentialWrapper implements AwsCredentialsProvider {
|
||||
|
||||
private final AWSCredentialsProvider oldCredentialsProvider;
|
||||
|
||||
@Override
|
||||
public AwsCredentials resolveCredentials() {
|
||||
AWSCredentials current = oldCredentialsProvider.getCredentials();
|
||||
if (current instanceof AWSSessionCredentials) {
|
||||
return AwsSessionCredentials.create(current.getAWSAccessKeyId(), current.getAWSSecretKey(),
|
||||
((AWSSessionCredentials) current).getSessionToken());
|
||||
}
|
||||
return new AwsCredentials() {
|
||||
@Override
|
||||
public String accessKeyId() {
|
||||
return current.getAWSAccessKeyId();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String secretAccessKey() {
|
||||
return current.getAWSSecretKey();
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
@ -36,6 +36,7 @@ public class CheckpointMessage extends Message {
|
|||
* The checkpoint this message is about.
|
||||
*/
|
||||
private String sequenceNumber;
|
||||
|
||||
private Long subSequenceNumber;
|
||||
|
||||
/**
|
||||
|
|
@ -61,5 +62,4 @@ public class CheckpointMessage extends Message {
|
|||
this.setError(throwable.getClass().getSimpleName());
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -33,14 +33,14 @@ public class InitializeMessage extends Message {
|
|||
* The shard id that this processor is getting initialized for.
|
||||
*/
|
||||
private String shardId;
|
||||
|
||||
private String sequenceNumber;
|
||||
private Long subSequenceNumber;
|
||||
|
||||
/**
|
||||
* Default constructor.
|
||||
*/
|
||||
public InitializeMessage() {
|
||||
}
|
||||
public InitializeMessage() {}
|
||||
|
||||
/**
|
||||
* Convenience constructor.
|
||||
|
|
@ -51,11 +51,11 @@ public class InitializeMessage extends Message {
|
|||
this.shardId = initializationInput.shardId();
|
||||
if (initializationInput.extendedSequenceNumber() != null) {
|
||||
this.sequenceNumber = initializationInput.extendedSequenceNumber().sequenceNumber();
|
||||
this.subSequenceNumber = initializationInput.extendedSequenceNumber().subSequenceNumber();
|
||||
this.subSequenceNumber =
|
||||
initializationInput.extendedSequenceNumber().subSequenceNumber();
|
||||
} else {
|
||||
this.sequenceNumber = null;
|
||||
this.subSequenceNumber = null;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@
|
|||
package software.amazon.kinesis.multilang.messages;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.Getter;
|
||||
|
|
@ -54,10 +53,11 @@ public class JsonFriendlyRecord {
|
|||
data = new byte[record.data().limit()];
|
||||
record.data().get(data);
|
||||
}
|
||||
Long approximateArrival = record.approximateArrivalTimestamp() == null ? null
|
||||
Long approximateArrival = record.approximateArrivalTimestamp() == null
|
||||
? null
|
||||
: record.approximateArrivalTimestamp().toEpochMilli();
|
||||
return new JsonFriendlyRecord(data, record.partitionKey(), record.sequenceNumber(),
|
||||
approximateArrival, record.subSequenceNumber());
|
||||
return new JsonFriendlyRecord(
|
||||
data, record.partitionKey(), record.sequenceNumber(), approximateArrival, record.subSequenceNumber());
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
|
|
|
|||
|
|
@ -21,5 +21,4 @@ package software.amazon.kinesis.multilang.messages;
|
|||
public class LeaseLostMessage extends Message {
|
||||
|
||||
public static final String ACTION = "leaseLost";
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -40,8 +40,7 @@ public abstract class Message {
|
|||
/**
|
||||
* Default constructor.
|
||||
*/
|
||||
public Message() {
|
||||
}
|
||||
public Message() {}
|
||||
|
||||
/**
|
||||
*
|
||||
|
|
|
|||
|
|
@ -37,13 +37,13 @@ public class ProcessRecordsMessage extends Message {
|
|||
* The records that the client's process needs to handle.
|
||||
*/
|
||||
private List<JsonFriendlyRecord> records;
|
||||
|
||||
private Long millisBehindLatest;
|
||||
|
||||
/**
|
||||
* Default constructor.
|
||||
*/
|
||||
public ProcessRecordsMessage() {
|
||||
}
|
||||
public ProcessRecordsMessage() {}
|
||||
|
||||
/**
|
||||
* Convenience constructor.
|
||||
|
|
|
|||
|
|
@ -145,4 +145,3 @@
|
|||
*
|
||||
*/
|
||||
package software.amazon.kinesis.multilang;
|
||||
|
||||
|
|
|
|||
|
|
@ -14,15 +14,14 @@
|
|||
*/
|
||||
package software.amazon.kinesis.multilang;
|
||||
|
||||
import static org.hamcrest.CoreMatchers.equalTo;
|
||||
import static org.hamcrest.CoreMatchers.nullValue;
|
||||
|
||||
import org.hamcrest.Description;
|
||||
import org.hamcrest.Matcher;
|
||||
import org.hamcrest.TypeSafeDiagnosingMatcher;
|
||||
|
||||
import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber;
|
||||
import software.amazon.kinesis.lifecycle.events.InitializationInput;
|
||||
import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber;
|
||||
|
||||
import static org.hamcrest.CoreMatchers.equalTo;
|
||||
import static org.hamcrest.CoreMatchers.nullValue;
|
||||
|
||||
public class Matchers {
|
||||
|
||||
|
|
@ -58,8 +57,12 @@ public class Matchers {
|
|||
|
||||
@Override
|
||||
public void describeTo(Description description) {
|
||||
description.appendText("An InitializationInput matching: { shardId: ").appendDescriptionOf(shardIdMatcher)
|
||||
.appendText(", sequenceNumber: ").appendDescriptionOf(sequenceNumberMatcher).appendText(" }");
|
||||
description
|
||||
.appendText("An InitializationInput matching: { shardId: ")
|
||||
.appendDescriptionOf(shardIdMatcher)
|
||||
.appendText(", sequenceNumber: ")
|
||||
.appendDescriptionOf(sequenceNumberMatcher)
|
||||
.appendText(" }");
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -98,10 +101,11 @@ public class Matchers {
|
|||
|
||||
@Override
|
||||
public void describeTo(Description description) {
|
||||
description.appendText("An ExtendedSequenceNumber matching: { sequenceNumber: ")
|
||||
.appendDescriptionOf(sequenceNumberMatcher).appendText(", subSequenceNumber: ")
|
||||
description
|
||||
.appendText("An ExtendedSequenceNumber matching: { sequenceNumber: ")
|
||||
.appendDescriptionOf(sequenceNumberMatcher)
|
||||
.appendText(", subSequenceNumber: ")
|
||||
.appendDescriptionOf(subSequenceNumberMatcher);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,15 +22,14 @@ import java.util.concurrent.ExecutionException;
|
|||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.mockito.Mockito;
|
||||
import org.mockito.invocation.InvocationOnMock;
|
||||
import org.mockito.stubbing.Answer;
|
||||
|
||||
import software.amazon.kinesis.multilang.messages.Message;
|
||||
import software.amazon.kinesis.multilang.messages.StatusMessage;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
public class MessageReaderTest {
|
||||
|
||||
|
|
@ -85,7 +84,9 @@ public class MessageReaderTest {
|
|||
try {
|
||||
Message message = reader.getNextMessageFromSTDOUT().get();
|
||||
if (message instanceof StatusMessage) {
|
||||
Assert.assertEquals("The status message's responseFor field should have been correct", responseFor,
|
||||
Assert.assertEquals(
|
||||
"The status message's responseFor field should have been correct",
|
||||
responseFor,
|
||||
((StatusMessage) message).getResponseFor());
|
||||
}
|
||||
} catch (InterruptedException | ExecutionException e) {
|
||||
|
|
@ -127,14 +128,15 @@ public class MessageReaderTest {
|
|||
return "{\"action\":\"shutdown\",\"reason\":\"ZOMBIE\"}";
|
||||
}
|
||||
}
|
||||
}).when(bufferReader).readLine();
|
||||
})
|
||||
.when(bufferReader)
|
||||
.readLine();
|
||||
} catch (IOException e) {
|
||||
Assert.fail("There shouldn't be an exception while setting up this mock.");
|
||||
}
|
||||
|
||||
MessageReader reader =
|
||||
new MessageReader().initialize(bufferReader, SHARD_ID, new ObjectMapper(),
|
||||
Executors.newCachedThreadPool());
|
||||
MessageReader reader = new MessageReader()
|
||||
.initialize(bufferReader, SHARD_ID, new ObjectMapper(), Executors.newCachedThreadPool());
|
||||
|
||||
try {
|
||||
reader.getNextMessageFromSTDOUT().get();
|
||||
|
|
@ -165,7 +167,8 @@ public class MessageReaderTest {
|
|||
readTask.get();
|
||||
Assert.fail("The reading task should have failed due to an IOException.");
|
||||
} catch (InterruptedException e) {
|
||||
Assert.fail("The reading task should not have been interrupted. It should have failed due to an IOException.");
|
||||
Assert.fail(
|
||||
"The reading task should not have been interrupted. It should have failed due to an IOException.");
|
||||
} catch (ExecutionException e) {
|
||||
// Yay!!
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,21 +23,19 @@ import java.util.concurrent.ExecutionException;
|
|||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.rules.ExpectedException;
|
||||
import org.mockito.Mockito;
|
||||
|
||||
import software.amazon.kinesis.lifecycle.events.InitializationInput;
|
||||
import software.amazon.kinesis.lifecycle.events.LeaseLostInput;
|
||||
import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput;
|
||||
import software.amazon.kinesis.lifecycle.events.ShardEndedInput;
|
||||
import software.amazon.kinesis.multilang.messages.Message;
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
import software.amazon.kinesis.lifecycle.events.InitializationInput;
|
||||
import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput;
|
||||
import software.amazon.kinesis.retrieval.KinesisClientRecord;
|
||||
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
|
@ -65,8 +63,7 @@ public class MessageWriterTest {
|
|||
public void writeCheckpointMessageNoErrorTest() throws IOException, InterruptedException, ExecutionException {
|
||||
Future<Boolean> future = this.messageWriter.writeCheckpointMessageWithError("1234", 0L, null);
|
||||
future.get();
|
||||
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(),
|
||||
Mockito.anyInt());
|
||||
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt());
|
||||
verify(this.stream, Mockito.atLeastOnce()).flush();
|
||||
}
|
||||
|
||||
|
|
@ -74,42 +71,43 @@ public class MessageWriterTest {
|
|||
public void writeCheckpointMessageWithErrorTest() throws IOException, InterruptedException, ExecutionException {
|
||||
Future<Boolean> future = this.messageWriter.writeCheckpointMessageWithError("1234", 0L, new Throwable());
|
||||
future.get();
|
||||
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(),
|
||||
Mockito.anyInt());
|
||||
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt());
|
||||
verify(this.stream, Mockito.atLeastOnce()).flush();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void writeInitializeMessageTest() throws IOException, InterruptedException, ExecutionException {
|
||||
Future<Boolean> future = this.messageWriter.writeInitializeMessage(InitializationInput.builder().shardId(SHARD_ID).build());
|
||||
Future<Boolean> future = this.messageWriter.writeInitializeMessage(
|
||||
InitializationInput.builder().shardId(SHARD_ID).build());
|
||||
future.get();
|
||||
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(),
|
||||
Mockito.anyInt());
|
||||
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt());
|
||||
verify(this.stream, Mockito.atLeastOnce()).flush();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void writeProcessRecordsMessageTest() throws IOException, InterruptedException, ExecutionException {
|
||||
List<KinesisClientRecord> records = Arrays.asList(
|
||||
KinesisClientRecord.builder().data(ByteBuffer.wrap("kitten".getBytes())).partitionKey("some cats")
|
||||
.sequenceNumber("357234807854789057805").build(),
|
||||
KinesisClientRecord.builder().build()
|
||||
);
|
||||
Future<Boolean> future = this.messageWriter.writeProcessRecordsMessage(ProcessRecordsInput.builder().records(records).build());
|
||||
KinesisClientRecord.builder()
|
||||
.data(ByteBuffer.wrap("kitten".getBytes()))
|
||||
.partitionKey("some cats")
|
||||
.sequenceNumber("357234807854789057805")
|
||||
.build(),
|
||||
KinesisClientRecord.builder().build());
|
||||
Future<Boolean> future = this.messageWriter.writeProcessRecordsMessage(
|
||||
ProcessRecordsInput.builder().records(records).build());
|
||||
future.get();
|
||||
|
||||
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(),
|
||||
Mockito.anyInt());
|
||||
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt());
|
||||
verify(this.stream, Mockito.atLeastOnce()).flush();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void writeShutdownMessageTest() throws IOException, InterruptedException, ExecutionException {
|
||||
Future<Boolean> future = this.messageWriter.writeShardEndedMessage(ShardEndedInput.builder().build());
|
||||
Future<Boolean> future = this.messageWriter.writeShardEndedMessage(
|
||||
ShardEndedInput.builder().build());
|
||||
future.get();
|
||||
|
||||
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(),
|
||||
Mockito.anyInt());
|
||||
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt());
|
||||
verify(this.stream, Mockito.atLeastOnce()).flush();
|
||||
}
|
||||
|
||||
|
|
@ -118,15 +116,15 @@ public class MessageWriterTest {
|
|||
Future<Boolean> future = this.messageWriter.writeShutdownRequestedMessage();
|
||||
future.get();
|
||||
|
||||
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(),
|
||||
Mockito.anyInt());
|
||||
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt());
|
||||
verify(this.stream, Mockito.atLeastOnce()).flush();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void streamIOExceptionTest() throws IOException, InterruptedException, ExecutionException {
|
||||
Mockito.doThrow(IOException.class).when(stream).flush();
|
||||
Future<Boolean> initializeTask = this.messageWriter.writeInitializeMessage(InitializationInput.builder().shardId(SHARD_ID).build());
|
||||
Future<Boolean> initializeTask = this.messageWriter.writeInitializeMessage(
|
||||
InitializationInput.builder().shardId(SHARD_ID).build());
|
||||
Boolean result = initializeTask.get();
|
||||
Assert.assertNotNull(result);
|
||||
Assert.assertFalse(result);
|
||||
|
|
@ -152,7 +150,8 @@ public class MessageWriterTest {
|
|||
Assert.assertFalse(this.messageWriter.isOpen());
|
||||
try {
|
||||
// Any message should fail
|
||||
this.messageWriter.writeInitializeMessage(InitializationInput.builder().shardId(SHARD_ID).build());
|
||||
this.messageWriter.writeInitializeMessage(
|
||||
InitializationInput.builder().shardId(SHARD_ID).build());
|
||||
Assert.fail("MessageWriter should be closed and unable to write.");
|
||||
} catch (IllegalStateException e) {
|
||||
// This should happen.
|
||||
|
|
|
|||
|
|
@ -14,29 +14,28 @@
|
|||
*/
|
||||
package software.amazon.kinesis.multilang;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
|
||||
import software.amazon.awssdk.regions.Region;
|
||||
import junit.framework.Assert;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Mockito;
|
||||
import org.mockito.runners.MockitoJUnitRunner;
|
||||
|
||||
import junit.framework.Assert;
|
||||
import software.amazon.awssdk.auth.credentials.AwsCredentials;
|
||||
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
|
||||
import software.amazon.awssdk.regions.Region;
|
||||
import software.amazon.kinesis.multilang.config.KinesisClientLibConfigurator;
|
||||
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@RunWith(MockitoJUnitRunner.class)
|
||||
public class MultiLangDaemonConfigTest {
|
||||
private static final String FILENAME = "some.properties";
|
||||
private static final String FILENAME = "multilang.properties";
|
||||
private static final String EXE = "TestExe.exe";
|
||||
private static final String APPLICATION_NAME = MultiLangDaemonConfigTest.class.getSimpleName();
|
||||
private static final String STREAM_NAME = "fakeStream";
|
||||
|
|
@ -49,10 +48,11 @@ public class MultiLangDaemonConfigTest {
|
|||
|
||||
@Mock
|
||||
private AwsCredentialsProvider credentialsProvider;
|
||||
|
||||
@Mock
|
||||
private AwsCredentials creds;
|
||||
|
||||
private KinesisClientLibConfigurator configurator;
|
||||
private final KinesisClientLibConfigurator configurator = new KinesisClientLibConfigurator();
|
||||
private MultiLangDaemonConfig deamonConfig;
|
||||
|
||||
/**
|
||||
|
|
@ -62,15 +62,13 @@ public class MultiLangDaemonConfigTest {
|
|||
* @throws IOException
|
||||
*/
|
||||
public void setup(String streamName, String streamArn) throws IOException {
|
||||
|
||||
String properties = String.format("executableName = %s\n"
|
||||
String properties = String.format(
|
||||
"executableName = %s\n"
|
||||
+ "applicationName = %s\n"
|
||||
+ "AWSCredentialsProvider = DefaultAWSCredentialsProviderChain\n"
|
||||
+ "AwsCredentialsProvider = DefaultCredentialsProvider\n"
|
||||
+ "processingLanguage = malbolge\n"
|
||||
+ "regionName = %s\n",
|
||||
EXE,
|
||||
APPLICATION_NAME,
|
||||
"us-east-1");
|
||||
EXE, APPLICATION_NAME, "us-east-1");
|
||||
|
||||
if (streamName != null) {
|
||||
properties += String.format("streamName = %s\n", streamName);
|
||||
|
|
@ -80,13 +78,12 @@ public class MultiLangDaemonConfigTest {
|
|||
}
|
||||
classLoader = Mockito.mock(ClassLoader.class);
|
||||
|
||||
Mockito.doReturn(new ByteArrayInputStream(properties.getBytes())).when(classLoader)
|
||||
Mockito.doReturn(new ByteArrayInputStream(properties.getBytes()))
|
||||
.when(classLoader)
|
||||
.getResourceAsStream(FILENAME);
|
||||
|
||||
when(credentialsProvider.resolveCredentials()).thenReturn(creds);
|
||||
when(creds.accessKeyId()).thenReturn("cool-user");
|
||||
configurator = new KinesisClientLibConfigurator();
|
||||
|
||||
deamonConfig = new MultiLangDaemonConfig(FILENAME, classLoader, configurator);
|
||||
}
|
||||
|
||||
|
|
@ -185,10 +182,11 @@ public class MultiLangDaemonConfigTest {
|
|||
@Test
|
||||
public void testPropertyValidation() {
|
||||
String propertiesNoExecutableName = "applicationName = testApp \n" + "streamName = fakeStream \n"
|
||||
+ "AWSCredentialsProvider = DefaultAWSCredentialsProviderChain\n" + "processingLanguage = malbolge";
|
||||
+ "AwsCredentialsProvider = DefaultCredentialsProvider\n" + "processingLanguage = malbolge";
|
||||
ClassLoader classLoader = Mockito.mock(ClassLoader.class);
|
||||
|
||||
Mockito.doReturn(new ByteArrayInputStream(propertiesNoExecutableName.getBytes())).when(classLoader)
|
||||
Mockito.doReturn(new ByteArrayInputStream(propertiesNoExecutableName.getBytes()))
|
||||
.when(classLoader)
|
||||
.getResourceAsStream(FILENAME);
|
||||
|
||||
try {
|
||||
|
|
@ -201,4 +199,13 @@ public class MultiLangDaemonConfigTest {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Test the loading of a "real" properties file. This test should catch
|
||||
* any issues which might arise if there is a discrepancy between reality
|
||||
* and mocking.
|
||||
*/
|
||||
@Test
|
||||
public void testActualPropertiesFile() throws Exception {
|
||||
new MultiLangDaemonConfig(FILENAME);
|
||||
}
|
||||
}
|
||||
|
|
@ -14,6 +14,28 @@
|
|||
*/
|
||||
package software.amazon.kinesis.multilang;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Future;
|
||||
|
||||
import ch.qos.logback.classic.LoggerContext;
|
||||
import ch.qos.logback.classic.joran.JoranConfigurator;
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.rules.ExpectedException;
|
||||
import org.junit.rules.TemporaryFolder;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.runners.MockitoJUnitRunner;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import software.amazon.kinesis.coordinator.Scheduler;
|
||||
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
|
||||
|
||||
import static org.hamcrest.Matchers.containsString;
|
||||
import static org.hamcrest.Matchers.empty;
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
|
|
@ -27,46 +49,29 @@ import static org.mockito.Mockito.spy;
|
|||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Future;
|
||||
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.rules.ExpectedException;
|
||||
import org.junit.rules.TemporaryFolder;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.runners.MockitoJUnitRunner;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import ch.qos.logback.classic.LoggerContext;
|
||||
import ch.qos.logback.classic.joran.JoranConfigurator;
|
||||
import software.amazon.kinesis.coordinator.Scheduler;
|
||||
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
|
||||
|
||||
@RunWith(MockitoJUnitRunner.class)
|
||||
public class MultiLangDaemonTest {
|
||||
@Mock
|
||||
private Scheduler scheduler;
|
||||
|
||||
@Mock
|
||||
private MultiLangDaemonConfig config;
|
||||
|
||||
@Mock
|
||||
private ExecutorService executorService;
|
||||
|
||||
@Mock
|
||||
private Future<Integer> futureInteger;
|
||||
|
||||
@Mock
|
||||
private MultiLangDaemonConfiguration multiLangDaemonConfiguration;
|
||||
|
||||
@Mock
|
||||
private Runtime runtime;
|
||||
|
||||
@Rule
|
||||
public ExpectedException expectedException = ExpectedException.none();
|
||||
|
||||
@Rule
|
||||
public final TemporaryFolder temporaryFolder = new TemporaryFolder();
|
||||
|
||||
|
|
@ -124,7 +129,8 @@ public class MultiLangDaemonTest {
|
|||
LoggerContext loggerContext = spy((LoggerContext) LoggerFactory.getILoggerFactory());
|
||||
JoranConfigurator configurator = spy(new JoranConfigurator());
|
||||
|
||||
String logConfiguration = this.getClass().getClassLoader().getResource("logback.xml").getPath();
|
||||
String logConfiguration =
|
||||
this.getClass().getClassLoader().getResource("logback.xml").getPath();
|
||||
daemon.configureLogging(logConfiguration, loggerContext, configurator);
|
||||
|
||||
verify(loggerContext).reset();
|
||||
|
|
@ -151,7 +157,7 @@ public class MultiLangDaemonTest {
|
|||
|
||||
MultiLangDaemon.MultiLangDaemonArguments arguments = new MultiLangDaemon.MultiLangDaemonArguments();
|
||||
|
||||
daemon.propertiesFile(arguments);
|
||||
daemon.validateAndGetPropertiesFileName(arguments);
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -160,7 +166,7 @@ public class MultiLangDaemonTest {
|
|||
MultiLangDaemon.MultiLangDaemonArguments arguments = new MultiLangDaemon.MultiLangDaemonArguments();
|
||||
arguments.parameters = Collections.singletonList(expectedPropertiesFile);
|
||||
|
||||
String propertiesFile = daemon.propertiesFile(arguments);
|
||||
String propertiesFile = daemon.validateAndGetPropertiesFileName(arguments);
|
||||
|
||||
assertThat(propertiesFile, equalTo(expectedPropertiesFile));
|
||||
}
|
||||
|
|
@ -174,7 +180,7 @@ public class MultiLangDaemonTest {
|
|||
arguments.parameters = Collections.singletonList(propertiesArgument);
|
||||
arguments.propertiesFile = propertiesOptions;
|
||||
|
||||
String propertiesFile = daemon.propertiesFile(arguments);
|
||||
String propertiesFile = daemon.validateAndGetPropertiesFileName(arguments);
|
||||
|
||||
assertThat(propertiesFile, equalTo(propertiesOptions));
|
||||
}
|
||||
|
|
@ -187,7 +193,7 @@ public class MultiLangDaemonTest {
|
|||
MultiLangDaemon.MultiLangDaemonArguments arguments = new MultiLangDaemon.MultiLangDaemonArguments();
|
||||
arguments.parameters = Arrays.asList("parameter1", "parameter2");
|
||||
|
||||
daemon.propertiesFile(arguments);
|
||||
daemon.validateAndGetPropertiesFileName(arguments);
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
|
|||
|
|
@ -14,6 +14,42 @@
|
|||
*/
|
||||
package software.amazon.kinesis.multilang;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
|
||||
import com.google.common.util.concurrent.SettableFuture;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Mockito;
|
||||
import org.mockito.invocation.InvocationOnMock;
|
||||
import org.mockito.runners.MockitoJUnitRunner;
|
||||
import org.mockito.stubbing.Answer;
|
||||
import software.amazon.kinesis.exceptions.InvalidStateException;
|
||||
import software.amazon.kinesis.exceptions.KinesisClientLibDependencyException;
|
||||
import software.amazon.kinesis.exceptions.ShutdownException;
|
||||
import software.amazon.kinesis.exceptions.ThrottlingException;
|
||||
import software.amazon.kinesis.lifecycle.events.InitializationInput;
|
||||
import software.amazon.kinesis.lifecycle.events.LeaseLostInput;
|
||||
import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput;
|
||||
import software.amazon.kinesis.lifecycle.events.ShardEndedInput;
|
||||
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
|
||||
import software.amazon.kinesis.multilang.messages.CheckpointMessage;
|
||||
import software.amazon.kinesis.multilang.messages.LeaseLostMessage;
|
||||
import software.amazon.kinesis.multilang.messages.Message;
|
||||
import software.amazon.kinesis.multilang.messages.ProcessRecordsMessage;
|
||||
import software.amazon.kinesis.multilang.messages.ShardEndedMessage;
|
||||
import software.amazon.kinesis.multilang.messages.StatusMessage;
|
||||
import software.amazon.kinesis.processor.RecordProcessorCheckpointer;
|
||||
import software.amazon.kinesis.retrieval.KinesisClientRecord;
|
||||
|
||||
import static org.hamcrest.CoreMatchers.equalTo;
|
||||
import static org.junit.Assert.assertThat;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
|
@ -27,65 +63,35 @@ import static org.mockito.Mockito.timeout;
|
|||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Mockito;
|
||||
import org.mockito.invocation.InvocationOnMock;
|
||||
import org.mockito.runners.MockitoJUnitRunner;
|
||||
import org.mockito.stubbing.Answer;
|
||||
|
||||
import software.amazon.kinesis.exceptions.InvalidStateException;
|
||||
import software.amazon.kinesis.exceptions.KinesisClientLibDependencyException;
|
||||
import software.amazon.kinesis.exceptions.ShutdownException;
|
||||
import software.amazon.kinesis.exceptions.ThrottlingException;
|
||||
import software.amazon.kinesis.lifecycle.events.LeaseLostInput;
|
||||
import software.amazon.kinesis.lifecycle.events.ShardEndedInput;
|
||||
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
|
||||
import software.amazon.kinesis.multilang.messages.CheckpointMessage;
|
||||
import software.amazon.kinesis.multilang.messages.LeaseLostMessage;
|
||||
import software.amazon.kinesis.multilang.messages.Message;
|
||||
import software.amazon.kinesis.multilang.messages.ProcessRecordsMessage;
|
||||
import software.amazon.kinesis.multilang.messages.ShardEndedMessage;
|
||||
import software.amazon.kinesis.multilang.messages.StatusMessage;
|
||||
import com.google.common.util.concurrent.SettableFuture;
|
||||
|
||||
import software.amazon.kinesis.lifecycle.events.InitializationInput;
|
||||
import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput;
|
||||
import software.amazon.kinesis.processor.RecordProcessorCheckpointer;
|
||||
import software.amazon.kinesis.retrieval.KinesisClientRecord;
|
||||
|
||||
@RunWith(MockitoJUnitRunner.class)
|
||||
public class MultiLangProtocolTest {
|
||||
private static final List<KinesisClientRecord> EMPTY_RECORD_LIST = Collections.emptyList();
|
||||
|
||||
@Mock
|
||||
private MultiLangProtocol protocol;
|
||||
|
||||
@Mock
|
||||
private MessageWriter messageWriter;
|
||||
|
||||
@Mock
|
||||
private MessageReader messageReader;
|
||||
|
||||
private String shardId;
|
||||
|
||||
@Mock
|
||||
private RecordProcessorCheckpointer checkpointer;
|
||||
|
||||
@Mock
|
||||
private MultiLangDaemonConfiguration configuration;
|
||||
|
||||
@Before
|
||||
public void setup() {
|
||||
this.shardId = "shard-id-123";
|
||||
protocol = new MultiLangProtocolForTesting(messageReader, messageWriter,
|
||||
InitializationInput.builder().shardId(shardId).build(), configuration);
|
||||
protocol = new MultiLangProtocolForTesting(
|
||||
messageReader,
|
||||
messageWriter,
|
||||
InitializationInput.builder().shardId(shardId).build(),
|
||||
configuration);
|
||||
|
||||
when(configuration.getTimeoutInSeconds()).thenReturn(null);
|
||||
}
|
||||
|
|
@ -104,28 +110,32 @@ public class MultiLangProtocolTest {
|
|||
|
||||
@Test
|
||||
public void testInitialize() {
|
||||
when(messageWriter
|
||||
.writeInitializeMessage(argThat(Matchers.withInit(InitializationInput.builder()
|
||||
.shardId(shardId).build())))).thenReturn(buildFuture(true));
|
||||
when(messageReader.getNextMessageFromSTDOUT()).thenReturn(buildFuture(
|
||||
new StatusMessage("initialize"), Message.class));
|
||||
when(messageWriter.writeInitializeMessage(argThat(Matchers.withInit(
|
||||
InitializationInput.builder().shardId(shardId).build()))))
|
||||
.thenReturn(buildFuture(true));
|
||||
when(messageReader.getNextMessageFromSTDOUT())
|
||||
.thenReturn(buildFuture(new StatusMessage("initialize"), Message.class));
|
||||
assertThat(protocol.initialize(), equalTo(true));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testProcessRecords() {
|
||||
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class))).thenReturn(buildFuture(true));
|
||||
when(messageReader.getNextMessageFromSTDOUT()).thenReturn(buildFuture(
|
||||
new StatusMessage("processRecords"), Message.class));
|
||||
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class)))
|
||||
.thenReturn(buildFuture(true));
|
||||
when(messageReader.getNextMessageFromSTDOUT())
|
||||
.thenReturn(buildFuture(new StatusMessage("processRecords"), Message.class));
|
||||
|
||||
assertThat(protocol.processRecords(ProcessRecordsInput.builder().records(EMPTY_RECORD_LIST).build()),
|
||||
assertThat(
|
||||
protocol.processRecords(
|
||||
ProcessRecordsInput.builder().records(EMPTY_RECORD_LIST).build()),
|
||||
equalTo(true));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void leaseLostTest() {
|
||||
when(messageWriter.writeLeaseLossMessage(any(LeaseLostInput.class))).thenReturn(buildFuture(true));
|
||||
when(messageReader.getNextMessageFromSTDOUT()).thenReturn(buildFuture(new StatusMessage(LeaseLostMessage.ACTION), Message.class));
|
||||
when(messageReader.getNextMessageFromSTDOUT())
|
||||
.thenReturn(buildFuture(new StatusMessage(LeaseLostMessage.ACTION), Message.class));
|
||||
|
||||
assertThat(protocol.leaseLost(LeaseLostInput.builder().build()), equalTo(true));
|
||||
}
|
||||
|
|
@ -133,7 +143,8 @@ public class MultiLangProtocolTest {
|
|||
@Test
|
||||
public void shardEndedTest() {
|
||||
when(messageWriter.writeShardEndedMessage(any(ShardEndedInput.class))).thenReturn(buildFuture(true));
|
||||
when(messageReader.getNextMessageFromSTDOUT()).thenReturn(buildFuture(new StatusMessage(ShardEndedMessage.ACTION)));
|
||||
when(messageReader.getNextMessageFromSTDOUT())
|
||||
.thenReturn(buildFuture(new StatusMessage(ShardEndedMessage.ACTION)));
|
||||
|
||||
assertThat(protocol.shardEnded(ShardEndedInput.builder().build()), equalTo(true));
|
||||
}
|
||||
|
|
@ -141,12 +152,12 @@ public class MultiLangProtocolTest {
|
|||
@Test
|
||||
public void shutdownRequestedTest() {
|
||||
when(messageWriter.writeShutdownRequestedMessage()).thenReturn(buildFuture(true));
|
||||
when(messageReader.getNextMessageFromSTDOUT()).thenReturn(buildFuture(
|
||||
new StatusMessage("shutdownRequested"), Message.class));
|
||||
Mockito.doReturn(buildFuture(true)).when(messageWriter)
|
||||
.writeShutdownRequestedMessage();
|
||||
when(messageReader.getNextMessageFromSTDOUT())
|
||||
.thenReturn(buildFuture(new StatusMessage("shutdownRequested"), Message.class));
|
||||
Mockito.doReturn(buildFuture(true)).when(messageWriter).writeShutdownRequestedMessage();
|
||||
Mockito.doReturn(buildFuture(new StatusMessage("shutdownRequested")))
|
||||
.when(messageReader).getNextMessageFromSTDOUT();
|
||||
.when(messageReader)
|
||||
.getNextMessageFromSTDOUT();
|
||||
assertThat(protocol.shutdownRequested(null), equalTo(true));
|
||||
}
|
||||
|
||||
|
|
@ -168,16 +179,17 @@ public class MultiLangProtocolTest {
|
|||
}
|
||||
return buildFuture(message);
|
||||
}
|
||||
|
||||
}.init(messages);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testProcessRecordsWithCheckpoints() throws
|
||||
KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
|
||||
public void testProcessRecordsWithCheckpoints()
|
||||
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
|
||||
|
||||
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class))).thenReturn(buildFuture(true));
|
||||
when(messageWriter.writeCheckpointMessageWithError(anyString(), anyLong(), any(Throwable.class))).thenReturn(buildFuture(true));
|
||||
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class)))
|
||||
.thenReturn(buildFuture(true));
|
||||
when(messageWriter.writeCheckpointMessageWithError(anyString(), anyLong(), any(Throwable.class)))
|
||||
.thenReturn(buildFuture(true));
|
||||
when(messageReader.getNextMessageFromSTDOUT()).thenAnswer(buildMessageAnswers(new ArrayList<Message>() {
|
||||
{
|
||||
this.add(new CheckpointMessage("123", 0L, null));
|
||||
|
|
@ -192,8 +204,10 @@ public class MultiLangProtocolTest {
|
|||
}
|
||||
}));
|
||||
|
||||
boolean result = protocol.processRecords(ProcessRecordsInput.builder().records(EMPTY_RECORD_LIST)
|
||||
.checkpointer(checkpointer).build());
|
||||
boolean result = protocol.processRecords(ProcessRecordsInput.builder()
|
||||
.records(EMPTY_RECORD_LIST)
|
||||
.checkpointer(checkpointer)
|
||||
.build());
|
||||
|
||||
assertThat(result, equalTo(true));
|
||||
|
||||
|
|
@ -203,41 +217,52 @@ public class MultiLangProtocolTest {
|
|||
|
||||
@Test
|
||||
public void testProcessRecordsWithABadCheckpoint() {
|
||||
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class))).thenReturn(buildFuture(true));
|
||||
when(messageWriter.writeCheckpointMessageWithError(anyString(), anyLong(), any(Throwable.class))).thenReturn(buildFuture(false));
|
||||
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class)))
|
||||
.thenReturn(buildFuture(true));
|
||||
when(messageWriter.writeCheckpointMessageWithError(anyString(), anyLong(), any(Throwable.class)))
|
||||
.thenReturn(buildFuture(false));
|
||||
when(messageReader.getNextMessageFromSTDOUT()).thenAnswer(buildMessageAnswers(new ArrayList<Message>() {
|
||||
{
|
||||
this.add(new CheckpointMessage("456", 0L, null));
|
||||
this.add(new StatusMessage("processRecords"));
|
||||
}
|
||||
}));
|
||||
assertThat(protocol.processRecords(ProcessRecordsInput.builder().records(EMPTY_RECORD_LIST)
|
||||
.checkpointer(checkpointer).build()), equalTo(false));
|
||||
assertThat(
|
||||
protocol.processRecords(ProcessRecordsInput.builder()
|
||||
.records(EMPTY_RECORD_LIST)
|
||||
.checkpointer(checkpointer)
|
||||
.build()),
|
||||
equalTo(false));
|
||||
}
|
||||
|
||||
@Test(expected = NullPointerException.class)
|
||||
public void waitForStatusMessageTimeoutTest() throws InterruptedException, TimeoutException, ExecutionException {
|
||||
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class))).thenReturn(buildFuture(true));
|
||||
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class)))
|
||||
.thenReturn(buildFuture(true));
|
||||
Future<Message> future = Mockito.mock(Future.class);
|
||||
when(messageReader.getNextMessageFromSTDOUT()).thenReturn(future);
|
||||
when(configuration.getTimeoutInSeconds()).thenReturn(5);
|
||||
when(future.get(anyInt(), eq(TimeUnit.SECONDS))).thenThrow(TimeoutException.class);
|
||||
protocol = new MultiLangProtocolForTesting(messageReader,
|
||||
protocol = new MultiLangProtocolForTesting(
|
||||
messageReader,
|
||||
messageWriter,
|
||||
InitializationInput.builder().shardId(shardId).build(),
|
||||
configuration);
|
||||
|
||||
protocol.processRecords(ProcessRecordsInput.builder().records(EMPTY_RECORD_LIST).build());
|
||||
protocol.processRecords(
|
||||
ProcessRecordsInput.builder().records(EMPTY_RECORD_LIST).build());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void waitForStatusMessageSuccessTest() {
|
||||
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class))).thenReturn(buildFuture(true));
|
||||
when(messageReader.getNextMessageFromSTDOUT()).thenReturn(buildFuture(
|
||||
new StatusMessage("processRecords"), Message.class));
|
||||
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class)))
|
||||
.thenReturn(buildFuture(true));
|
||||
when(messageReader.getNextMessageFromSTDOUT())
|
||||
.thenReturn(buildFuture(new StatusMessage("processRecords"), Message.class));
|
||||
when(configuration.getTimeoutInSeconds()).thenReturn(5);
|
||||
|
||||
assertTrue(protocol.processRecords(ProcessRecordsInput.builder().records(EMPTY_RECORD_LIST).build()));
|
||||
assertTrue(protocol.processRecords(
|
||||
ProcessRecordsInput.builder().records(EMPTY_RECORD_LIST).build()));
|
||||
}
|
||||
|
||||
private class MultiLangProtocolForTesting extends MultiLangProtocol {
|
||||
|
|
@ -249,7 +274,8 @@ public class MultiLangProtocolTest {
|
|||
* @param initializationInput
|
||||
* @param configuration
|
||||
*/
|
||||
MultiLangProtocolForTesting(final MessageReader messageReader,
|
||||
MultiLangProtocolForTesting(
|
||||
final MessageReader messageReader,
|
||||
final MessageWriter messageWriter,
|
||||
final InitializationInput initializationInput,
|
||||
final MultiLangDaemonConfiguration configuration) {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,110 @@
|
|||
/*
|
||||
* Copyright 2023 Amazon.com, Inc. or its affiliates.
|
||||
* Licensed under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package software.amazon.kinesis.multilang;
|
||||
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.runners.MockitoJUnitRunner;
|
||||
import software.amazon.awssdk.regions.Region;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.verifyZeroInteractions;
|
||||
import static software.amazon.kinesis.multilang.NestedPropertyKey.ENDPOINT;
|
||||
import static software.amazon.kinesis.multilang.NestedPropertyKey.ENDPOINT_REGION;
|
||||
import static software.amazon.kinesis.multilang.NestedPropertyKey.EXTERNAL_ID;
|
||||
import static software.amazon.kinesis.multilang.NestedPropertyKey.parse;
|
||||
|
||||
@RunWith(MockitoJUnitRunner.class)
|
||||
public class NestedPropertyKeyTest {
|
||||
|
||||
@Mock
|
||||
private NestedPropertyProcessor mockProcessor;
|
||||
|
||||
@Test
|
||||
public void testExternalId() {
|
||||
final String expectedId = "eid";
|
||||
|
||||
parse(mockProcessor, createKey(EXTERNAL_ID, expectedId));
|
||||
verify(mockProcessor).acceptExternalId(expectedId);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testEndpoint() {
|
||||
final String expectedEndpoint = "https://sts.us-east-1.amazonaws.com";
|
||||
final String expectedRegion = "us-east-1";
|
||||
final String param = createKey(ENDPOINT, expectedEndpoint + "^" + expectedRegion);
|
||||
|
||||
parse(mockProcessor, param);
|
||||
verify(mockProcessor).acceptEndpoint(expectedEndpoint, expectedRegion);
|
||||
}
|
||||
|
||||
@Test(expected = IllegalArgumentException.class)
|
||||
public void testInvalidEndpoint() {
|
||||
parse(mockProcessor, createKey(ENDPOINT, "value-sans-caret-delimiter"));
|
||||
}
|
||||
|
||||
@Test(expected = IllegalArgumentException.class)
|
||||
public void testInvalidEndpointDoubleCaret() {
|
||||
parse(mockProcessor, createKey(ENDPOINT, "https://sts.us-east-1.amazonaws.com^us-east-1^borkbork"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testEndpointRegion() {
|
||||
final Region expectedRegion = Region.US_GOV_WEST_1;
|
||||
|
||||
parse(mockProcessor, createKey(ENDPOINT_REGION, expectedRegion.id()));
|
||||
verify(mockProcessor).acceptEndpointRegion(expectedRegion);
|
||||
}
|
||||
|
||||
@Test(expected = IllegalArgumentException.class)
|
||||
public void testInvalidEndpointRegion() {
|
||||
parse(mockProcessor, createKey(ENDPOINT_REGION, "snuffleupagus"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Test that the literal nested key (i.e., {@code key=} in {@code some_val|key=nested_val})
|
||||
* does not change. Any change to an existing literal key is not backwards-compatible.
|
||||
*/
|
||||
@Test
|
||||
public void testKeysExplicitly() {
|
||||
// Adding a new enum will deliberately cause this assert to fail, and
|
||||
// therefore raise awareness for this explicit test. Add-and-remove may
|
||||
// keep the number unchanged yet will also break (by removing an enum).
|
||||
assertEquals(3, NestedPropertyKey.values().length);
|
||||
|
||||
assertEquals("endpoint", ENDPOINT.getNestedKey());
|
||||
assertEquals("endpointRegion", ENDPOINT_REGION.getNestedKey());
|
||||
assertEquals("externalId", EXTERNAL_ID.getNestedKey());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNonmatchingParameters() {
|
||||
final String[] params = new String[] {
|
||||
null,
|
||||
"",
|
||||
"hello world", // no nested key
|
||||
"foo=bar", // nested key, but is not a recognized key
|
||||
createKey(EXTERNAL_ID, "eid") + "=extra", // valid key made invalid by second '='
|
||||
};
|
||||
parse(mockProcessor, params);
|
||||
verifyZeroInteractions(mockProcessor);
|
||||
}
|
||||
|
||||
private static String createKey(final NestedPropertyKey key, final String value) {
|
||||
return key.getNestedKey() + "=" + value;
|
||||
}
|
||||
}
|
||||
|
|
@ -72,7 +72,8 @@ public class ReadSTDERRTaskTest {
|
|||
try {
|
||||
finishedCleanly = result.get();
|
||||
} catch (InterruptedException | ExecutionException e) {
|
||||
Assert.fail("Should have been able to get a result. The error should be handled during the call and result in false.");
|
||||
Assert.fail(
|
||||
"Should have been able to get a result. The error should be handled during the call and result in false.");
|
||||
}
|
||||
Assert.assertFalse("Reading a line should have thrown an exception", finishedCleanly);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,12 +16,11 @@ package software.amazon.kinesis.multilang;
|
|||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
|
||||
import software.amazon.kinesis.processor.ShardRecordProcessor;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.runners.MockitoJUnitRunner;
|
||||
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
|
||||
import software.amazon.kinesis.processor.ShardRecordProcessor;
|
||||
|
||||
@RunWith(MockitoJUnitRunner.class)
|
||||
public class StreamingShardRecordProcessorFactoryTest {
|
||||
|
|
@ -31,10 +30,13 @@ public class StreamingShardRecordProcessorFactoryTest {
|
|||
|
||||
@Test
|
||||
public void createProcessorTest() {
|
||||
MultiLangRecordProcessorFactory factory = new MultiLangRecordProcessorFactory("somecommand", null, configuration);
|
||||
MultiLangRecordProcessorFactory factory =
|
||||
new MultiLangRecordProcessorFactory("somecommand", null, configuration);
|
||||
ShardRecordProcessor processor = factory.shardRecordProcessor();
|
||||
|
||||
Assert.assertEquals("Should have constructed a StreamingRecordProcessor", MultiLangShardRecordProcessor.class,
|
||||
Assert.assertEquals(
|
||||
"Should have constructed a StreamingRecordProcessor",
|
||||
MultiLangShardRecordProcessor.class,
|
||||
processor.getClass());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,15 +14,6 @@
|
|||
*/
|
||||
package software.amazon.kinesis.multilang;
|
||||
|
||||
import static org.mockito.Matchers.any;
|
||||
import static org.mockito.Matchers.anyLong;
|
||||
import static org.mockito.Matchers.anyString;
|
||||
import static org.mockito.Matchers.argThat;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.util.Collections;
|
||||
|
|
@ -32,6 +23,7 @@ import java.util.concurrent.ExecutorService;
|
|||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
|
@ -41,9 +33,6 @@ import org.mockito.Mockito;
|
|||
import org.mockito.invocation.InvocationOnMock;
|
||||
import org.mockito.runners.MockitoJUnitRunner;
|
||||
import org.mockito.stubbing.Answer;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
import software.amazon.awssdk.services.kinesis.model.Record;
|
||||
import software.amazon.kinesis.exceptions.KinesisClientLibDependencyException;
|
||||
import software.amazon.kinesis.exceptions.ThrottlingException;
|
||||
|
|
@ -61,6 +50,15 @@ import software.amazon.kinesis.processor.PreparedCheckpointer;
|
|||
import software.amazon.kinesis.processor.RecordProcessorCheckpointer;
|
||||
import software.amazon.kinesis.retrieval.KinesisClientRecord;
|
||||
|
||||
import static org.mockito.Matchers.any;
|
||||
import static org.mockito.Matchers.anyLong;
|
||||
import static org.mockito.Matchers.anyString;
|
||||
import static org.mockito.Matchers.argThat;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@RunWith(MockitoJUnitRunner.class)
|
||||
public class StreamingShardRecordProcessorTest {
|
||||
|
||||
|
|
@ -70,6 +68,7 @@ public class StreamingShardRecordProcessorTest {
|
|||
|
||||
@Mock
|
||||
private Future<Message> messageFuture;
|
||||
|
||||
@Mock
|
||||
private Future<Boolean> trueFuture;
|
||||
|
||||
|
|
@ -81,14 +80,13 @@ public class StreamingShardRecordProcessorTest {
|
|||
}
|
||||
|
||||
@Override
|
||||
public void checkpoint(String sequenceNumber) throws KinesisClientLibDependencyException,
|
||||
ThrottlingException, IllegalArgumentException {
|
||||
public void checkpoint(String sequenceNumber)
|
||||
throws KinesisClientLibDependencyException, ThrottlingException, IllegalArgumentException {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void checkpoint(Record record)
|
||||
throws KinesisClientLibDependencyException, ThrottlingException {
|
||||
public void checkpoint(Record record) throws KinesisClientLibDependencyException, ThrottlingException {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
|
|
@ -141,7 +139,8 @@ public class StreamingShardRecordProcessorTest {
|
|||
}
|
||||
|
||||
@Override
|
||||
public PreparedCheckpointer prepareCheckpoint(String sequenceNumber, long subSequenceNumber, byte[] applicationState)
|
||||
public PreparedCheckpointer prepareCheckpoint(
|
||||
String sequenceNumber, long subSequenceNumber, byte[] applicationState)
|
||||
throws KinesisClientLibDependencyException, ThrottlingException, IllegalArgumentException {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
|
@ -178,8 +177,14 @@ public class StreamingShardRecordProcessorTest {
|
|||
when(configuration.getTimeoutInSeconds()).thenReturn(null);
|
||||
|
||||
recordProcessor =
|
||||
new MultiLangShardRecordProcessor(new ProcessBuilder(), executor, new ObjectMapper(), messageWriter,
|
||||
messageReader, errorReader, configuration) {
|
||||
new MultiLangShardRecordProcessor(
|
||||
new ProcessBuilder(),
|
||||
executor,
|
||||
new ObjectMapper(),
|
||||
messageWriter,
|
||||
messageReader,
|
||||
errorReader,
|
||||
configuration) {
|
||||
|
||||
// Just don't do anything when we exit.
|
||||
void exit() {
|
||||
|
|
@ -203,9 +208,12 @@ public class StreamingShardRecordProcessorTest {
|
|||
Mockito.doReturn(Mockito.mock(Future.class)).when(messageReader).drainSTDOUT();
|
||||
Mockito.doReturn(true).when(trueFuture).get();
|
||||
|
||||
when(messageWriter.writeInitializeMessage(any(InitializationInput.class))).thenReturn(trueFuture);
|
||||
when(messageWriter.writeCheckpointMessageWithError(anyString(), anyLong(), any(Throwable.class))).thenReturn(trueFuture);
|
||||
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class))).thenReturn(trueFuture);
|
||||
when(messageWriter.writeInitializeMessage(any(InitializationInput.class)))
|
||||
.thenReturn(trueFuture);
|
||||
when(messageWriter.writeCheckpointMessageWithError(anyString(), anyLong(), any(Throwable.class)))
|
||||
.thenReturn(trueFuture);
|
||||
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class)))
|
||||
.thenReturn(trueFuture);
|
||||
when(messageWriter.writeLeaseLossMessage(any(LeaseLostInput.class))).thenReturn(trueFuture);
|
||||
}
|
||||
|
||||
|
|
@ -223,11 +231,16 @@ public class StreamingShardRecordProcessorTest {
|
|||
|
||||
List<KinesisClientRecord> testRecords = Collections.emptyList();
|
||||
|
||||
recordProcessor.initialize(InitializationInput.builder().shardId(SHARD_ID).build());
|
||||
recordProcessor.processRecords(ProcessRecordsInput.builder().records(testRecords)
|
||||
.checkpointer(unimplementedCheckpointer).build());
|
||||
recordProcessor.processRecords(ProcessRecordsInput.builder().records(testRecords)
|
||||
.checkpointer(unimplementedCheckpointer).build());
|
||||
recordProcessor.initialize(
|
||||
InitializationInput.builder().shardId(SHARD_ID).build());
|
||||
recordProcessor.processRecords(ProcessRecordsInput.builder()
|
||||
.records(testRecords)
|
||||
.checkpointer(unimplementedCheckpointer)
|
||||
.build());
|
||||
recordProcessor.processRecords(ProcessRecordsInput.builder()
|
||||
.records(testRecords)
|
||||
.checkpointer(unimplementedCheckpointer)
|
||||
.build());
|
||||
recordProcessor.leaseLost(LeaseLostInput.builder().build());
|
||||
}
|
||||
|
||||
|
|
@ -235,9 +248,12 @@ public class StreamingShardRecordProcessorTest {
|
|||
public void processorPhasesTest() throws InterruptedException, ExecutionException {
|
||||
Answer<StatusMessage> answer = new Answer<StatusMessage>() {
|
||||
|
||||
StatusMessage[] answers = new StatusMessage[] { new StatusMessage(InitializeMessage.ACTION),
|
||||
new StatusMessage(ProcessRecordsMessage.ACTION), new StatusMessage(ProcessRecordsMessage.ACTION),
|
||||
new StatusMessage(ShutdownMessage.ACTION) };
|
||||
StatusMessage[] answers = new StatusMessage[] {
|
||||
new StatusMessage(InitializeMessage.ACTION),
|
||||
new StatusMessage(ProcessRecordsMessage.ACTION),
|
||||
new StatusMessage(ProcessRecordsMessage.ACTION),
|
||||
new StatusMessage(ShutdownMessage.ACTION)
|
||||
};
|
||||
|
||||
int callCount = 0;
|
||||
|
||||
|
|
@ -268,9 +284,12 @@ public class StreamingShardRecordProcessorTest {
|
|||
* This bad message will cause shutdown to not attempt to send a message. i.e. avoid encountering an
|
||||
* exception.
|
||||
*/
|
||||
StatusMessage[] answers = new StatusMessage[] { new StatusMessage("Bad"),
|
||||
new StatusMessage(ProcessRecordsMessage.ACTION), new StatusMessage(ProcessRecordsMessage.ACTION),
|
||||
new StatusMessage(ShutdownMessage.ACTION) };
|
||||
StatusMessage[] answers = new StatusMessage[] {
|
||||
new StatusMessage("Bad"),
|
||||
new StatusMessage(ProcessRecordsMessage.ACTION),
|
||||
new StatusMessage(ProcessRecordsMessage.ACTION),
|
||||
new StatusMessage(ShutdownMessage.ACTION)
|
||||
};
|
||||
|
||||
int callCount = 0;
|
||||
|
||||
|
|
@ -286,8 +305,9 @@ public class StreamingShardRecordProcessorTest {
|
|||
|
||||
phases(answer);
|
||||
|
||||
verify(messageWriter).writeInitializeMessage(argThat(Matchers.withInit(InitializationInput.builder()
|
||||
.shardId(SHARD_ID).build())));
|
||||
verify(messageWriter)
|
||||
.writeInitializeMessage(argThat(Matchers.withInit(
|
||||
InitializationInput.builder().shardId(SHARD_ID).build())));
|
||||
verify(messageWriter, times(2)).writeProcessRecordsMessage(any(ProcessRecordsInput.class));
|
||||
verify(messageWriter, never()).writeLeaseLossMessage(any(LeaseLostInput.class));
|
||||
Assert.assertEquals(1, systemExitCount);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,70 @@
|
|||
/*
|
||||
* Copyright 2023 Amazon.com, Inc. or its affiliates.
|
||||
* Licensed under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package software.amazon.kinesis.multilang.auth;
|
||||
|
||||
import java.util.Arrays;
|
||||
|
||||
import org.junit.Test;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
public class KclStsAssumeRoleCredentialsProviderTest {
|
||||
|
||||
private static final String ARN = "arn";
|
||||
private static final String SESSION_NAME = "sessionName";
|
||||
|
||||
/**
|
||||
* Test that the constructor doesn't throw an out-of-bounds exception if
|
||||
* there are no parameters beyond the required ARN and session name.
|
||||
*/
|
||||
@Test
|
||||
public void testConstructorWithoutOptionalParams() {
|
||||
new KclStsAssumeRoleCredentialsProvider(new String[] {ARN, SESSION_NAME, "endpointRegion=us-east-1"});
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAcceptEndpoint() {
|
||||
// discovered exception during e2e testing; therefore, this test is
|
||||
// to simply verify the constructed STS client doesn't go *boom*
|
||||
final KclStsAssumeRoleCredentialsProvider provider =
|
||||
new KclStsAssumeRoleCredentialsProvider(ARN, SESSION_NAME, "endpointRegion=us-east-1");
|
||||
provider.acceptEndpoint("endpoint", "us-east-1");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testVarArgs() {
|
||||
for (final String[] varargs : Arrays.asList(
|
||||
new String[] {ARN, SESSION_NAME, "externalId=eid", "foo", "endpointRegion=us-east-1"},
|
||||
new String[] {ARN, SESSION_NAME, "foo", "externalId=eid", "endpointRegion=us-east-1"})) {
|
||||
final VarArgsSpy provider = new VarArgsSpy(varargs);
|
||||
assertEquals("eid", provider.externalId);
|
||||
}
|
||||
}
|
||||
|
||||
private static class VarArgsSpy extends KclStsAssumeRoleCredentialsProvider {
|
||||
|
||||
private String externalId;
|
||||
|
||||
public VarArgsSpy(String[] args) {
|
||||
super(args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void acceptExternalId(final String externalId) {
|
||||
this.externalId = externalId;
|
||||
super.acceptExternalId(externalId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,161 +0,0 @@
|
|||
/*
|
||||
* Copyright 2019 Amazon.com, Inc. or its affiliates.
|
||||
* Licensed under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package software.amazon.kinesis.multilang.config;
|
||||
|
||||
import static org.hamcrest.CoreMatchers.equalTo;
|
||||
import static org.hamcrest.CoreMatchers.instanceOf;
|
||||
import static org.junit.Assert.assertThat;
|
||||
|
||||
import java.util.Arrays;
|
||||
|
||||
import com.amazonaws.auth.BasicAWSCredentials;
|
||||
import lombok.ToString;
|
||||
import org.hamcrest.Description;
|
||||
import org.hamcrest.Matcher;
|
||||
import org.hamcrest.TypeSafeDiagnosingMatcher;
|
||||
import org.junit.Test;
|
||||
|
||||
import com.amazonaws.auth.AWSCredentials;
|
||||
import com.amazonaws.auth.AWSCredentialsProvider;
|
||||
import com.amazonaws.auth.AWSCredentialsProviderChain;
|
||||
|
||||
public class AWSCredentialsProviderPropertyValueDecoderTest {
|
||||
|
||||
private static final String TEST_ACCESS_KEY_ID = "123";
|
||||
private static final String TEST_SECRET_KEY = "456";
|
||||
|
||||
private final String credentialName1 = AlwaysSucceedCredentialsProvider.class.getName();
|
||||
private final String credentialName2 = ConstructorCredentialsProvider.class.getName();
|
||||
private AWSCredentialsProviderPropertyValueDecoder decoder = new AWSCredentialsProviderPropertyValueDecoder();
|
||||
|
||||
@ToString
|
||||
private static class AWSCredentialsMatcher extends TypeSafeDiagnosingMatcher<AWSCredentialsProvider> {
|
||||
|
||||
private final Matcher<String> akidMatcher;
|
||||
private final Matcher<String> secretMatcher;
|
||||
private final Matcher<Class<?>> classMatcher;
|
||||
|
||||
public AWSCredentialsMatcher(String akid, String secret) {
|
||||
this.akidMatcher = equalTo(akid);
|
||||
this.secretMatcher = equalTo(secret);
|
||||
this.classMatcher = instanceOf(AWSCredentialsProviderChain.class);
|
||||
}
|
||||
|
||||
private AWSCredentialsMatcher(AWSCredentials expected) {
|
||||
this(expected.getAWSAccessKeyId(), expected.getAWSSecretKey());
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean matchesSafely(AWSCredentialsProvider item, Description mismatchDescription) {
|
||||
AWSCredentials actual = item.getCredentials();
|
||||
boolean matched = true;
|
||||
|
||||
if (!classMatcher.matches(item)) {
|
||||
classMatcher.describeMismatch(item, mismatchDescription);
|
||||
matched = false;
|
||||
}
|
||||
|
||||
if (!akidMatcher.matches(actual.getAWSAccessKeyId())) {
|
||||
akidMatcher.describeMismatch(actual.getAWSAccessKeyId(), mismatchDescription);
|
||||
matched = false;
|
||||
}
|
||||
if (!secretMatcher.matches(actual.getAWSSecretKey())) {
|
||||
secretMatcher.describeMismatch(actual.getAWSSecretKey(), mismatchDescription);
|
||||
matched = false;
|
||||
}
|
||||
return matched;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void describeTo(Description description) {
|
||||
description.appendText("An AWSCredentialsProvider that provides an AWSCredential matching: ")
|
||||
.appendList("(", ", ", ")", Arrays.asList(classMatcher, akidMatcher, secretMatcher));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private static AWSCredentialsMatcher hasCredentials(String akid, String secret) {
|
||||
return new AWSCredentialsMatcher(akid, secret);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSingleProvider() {
|
||||
AWSCredentialsProvider provider = decoder.decodeValue(credentialName1);
|
||||
assertThat(provider, hasCredentials(TEST_ACCESS_KEY_ID, TEST_SECRET_KEY));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testTwoProviders() {
|
||||
AWSCredentialsProvider provider = decoder.decodeValue(credentialName1 + "," + credentialName1);
|
||||
assertThat(provider, hasCredentials(TEST_ACCESS_KEY_ID, TEST_SECRET_KEY));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testProfileProviderWithOneArg() {
|
||||
AWSCredentialsProvider provider = decoder.decodeValue(credentialName2 + "|arg");
|
||||
assertThat(provider, hasCredentials("arg", "blank"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testProfileProviderWithTwoArgs() {
|
||||
AWSCredentialsProvider provider = decoder.decodeValue(credentialName2 + "|arg1|arg2");
|
||||
assertThat(provider, hasCredentials("arg1", "arg2"));
|
||||
}
|
||||
|
||||
/**
|
||||
* This credentials provider will always succeed
|
||||
*/
|
||||
public static class AlwaysSucceedCredentialsProvider implements AWSCredentialsProvider {
|
||||
|
||||
@Override
|
||||
public AWSCredentials getCredentials() {
|
||||
return new BasicAWSCredentials(TEST_ACCESS_KEY_ID, TEST_SECRET_KEY);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void refresh() {
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This credentials provider needs a constructor call to instantiate it
|
||||
*/
|
||||
public static class ConstructorCredentialsProvider implements AWSCredentialsProvider {
|
||||
|
||||
private String arg1;
|
||||
private String arg2;
|
||||
|
||||
public ConstructorCredentialsProvider(String arg1) {
|
||||
this.arg1 = arg1;
|
||||
this.arg2 = "blank";
|
||||
}
|
||||
|
||||
public ConstructorCredentialsProvider(String arg1, String arg2) {
|
||||
this.arg1 = arg1;
|
||||
this.arg2 = arg2;
|
||||
}
|
||||
|
||||
@Override
|
||||
public AWSCredentials getCredentials() {
|
||||
return new BasicAWSCredentials(arg1, arg2);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void refresh() {
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,237 @@
|
|||
/*
|
||||
* Copyright 2019 Amazon.com, Inc. or its affiliates.
|
||||
* Licensed under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package software.amazon.kinesis.multilang.config;
|
||||
|
||||
import java.util.Arrays;
|
||||
|
||||
import lombok.ToString;
|
||||
import org.hamcrest.Description;
|
||||
import org.hamcrest.Matcher;
|
||||
import org.hamcrest.TypeSafeDiagnosingMatcher;
|
||||
import org.junit.Test;
|
||||
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
|
||||
import software.amazon.awssdk.auth.credentials.AwsCredentials;
|
||||
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
|
||||
import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain;
|
||||
import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider;
|
||||
import software.amazon.kinesis.multilang.auth.KclStsAssumeRoleCredentialsProvider;
|
||||
|
||||
import static org.hamcrest.CoreMatchers.equalTo;
|
||||
import static org.hamcrest.CoreMatchers.instanceOf;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
import static org.junit.Assert.assertThat;
|
||||
|
||||
public class AwsCredentialsProviderPropertyValueDecoderTest {
|
||||
|
||||
private static final String TEST_ACCESS_KEY_ID = "123";
|
||||
private static final String TEST_SECRET_KEY = "456";
|
||||
|
||||
private final String credentialName1 = AlwaysSucceedCredentialsProvider.class.getName();
|
||||
private final String credentialName2 = ConstructorCredentialsProvider.class.getName();
|
||||
private final String createCredentialClass = CreateProvider.class.getName();
|
||||
private final AwsCredentialsProviderPropertyValueDecoder decoder = new AwsCredentialsProviderPropertyValueDecoder();
|
||||
|
||||
@ToString
|
||||
private static class AwsCredentialsMatcher extends TypeSafeDiagnosingMatcher<AwsCredentialsProvider> {
|
||||
|
||||
private final Matcher<String> akidMatcher;
|
||||
private final Matcher<String> secretMatcher;
|
||||
private final Matcher<Class<?>> classMatcher;
|
||||
|
||||
public AwsCredentialsMatcher(String akid, String secret) {
|
||||
this.akidMatcher = equalTo(akid);
|
||||
this.secretMatcher = equalTo(secret);
|
||||
this.classMatcher = instanceOf(AwsCredentialsProviderChain.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean matchesSafely(AwsCredentialsProvider item, Description mismatchDescription) {
|
||||
AwsCredentials actual = item.resolveCredentials();
|
||||
boolean matched = true;
|
||||
|
||||
if (!classMatcher.matches(item)) {
|
||||
classMatcher.describeMismatch(item, mismatchDescription);
|
||||
matched = false;
|
||||
}
|
||||
|
||||
if (!akidMatcher.matches(actual.accessKeyId())) {
|
||||
akidMatcher.describeMismatch(actual.accessKeyId(), mismatchDescription);
|
||||
matched = false;
|
||||
}
|
||||
if (!secretMatcher.matches(actual.secretAccessKey())) {
|
||||
secretMatcher.describeMismatch(actual.secretAccessKey(), mismatchDescription);
|
||||
matched = false;
|
||||
}
|
||||
return matched;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void describeTo(Description description) {
|
||||
description
|
||||
.appendText("An AwsCredentialsProvider that provides an AwsCredential matching: ")
|
||||
.appendList("(", ", ", ")", Arrays.asList(classMatcher, akidMatcher, secretMatcher));
|
||||
}
|
||||
}
|
||||
|
||||
private static AwsCredentialsMatcher hasCredentials(String akid, String secret) {
|
||||
return new AwsCredentialsMatcher(akid, secret);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSingleProvider() {
|
||||
AwsCredentialsProvider provider = decoder.decodeValue(credentialName1);
|
||||
assertThat(provider, hasCredentials(TEST_ACCESS_KEY_ID, TEST_SECRET_KEY));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testTwoProviders() {
|
||||
AwsCredentialsProvider provider = decoder.decodeValue(credentialName1 + "," + credentialName1);
|
||||
assertThat(provider, hasCredentials(TEST_ACCESS_KEY_ID, TEST_SECRET_KEY));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testProfileProviderWithOneArg() {
|
||||
AwsCredentialsProvider provider = decoder.decodeValue(credentialName2 + "|arg");
|
||||
assertThat(provider, hasCredentials("arg", "blank"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testProfileProviderWithTwoArgs() {
|
||||
AwsCredentialsProvider provider = decoder.decodeValue(credentialName2 + "|arg1|arg2");
|
||||
assertThat(provider, hasCredentials("arg1", "arg2"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Test that providers in the multi-lang auth package can be resolved and instantiated.
|
||||
*/
|
||||
@Test
|
||||
public void testKclAuthProvider() {
|
||||
for (final String className : Arrays.asList(
|
||||
KclStsAssumeRoleCredentialsProvider.class.getName(), // fully-qualified name
|
||||
KclStsAssumeRoleCredentialsProvider.class.getSimpleName(), // name-only; needs prefix
|
||||
StsAssumeRoleCredentialsProvider.class.getName(), // user passes full sts package path
|
||||
StsAssumeRoleCredentialsProvider.class.getSimpleName())) {
|
||||
final AwsCredentialsProvider provider =
|
||||
decoder.decodeValue(className + "|arn|sessionName|endpointRegion=us-east-1");
|
||||
assertNotNull(className, provider);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Test that OneArgCreateProvider in the SDK v2 can process a create() method
|
||||
*/
|
||||
@Test
|
||||
public void testEmptyCreateProvider() {
|
||||
AwsCredentialsProvider provider = decoder.decodeValue(createCredentialClass);
|
||||
assertThat(provider, hasCredentials(TEST_ACCESS_KEY_ID, TEST_SECRET_KEY));
|
||||
}
|
||||
|
||||
/**
|
||||
* Test that OneArgCreateProvider in the SDK v2 can process a create(arg1) method
|
||||
*/
|
||||
@Test
|
||||
public void testOneArgCreateProvider() {
|
||||
AwsCredentialsProvider provider = decoder.decodeValue(createCredentialClass + "|testCreateProperty");
|
||||
assertThat(provider, hasCredentials("testCreateProperty", TEST_SECRET_KEY));
|
||||
}
|
||||
|
||||
/**
|
||||
* Test that a provider can be instantiated by its varargs constructor.
|
||||
*/
|
||||
@Test
|
||||
public void testVarArgAuthProvider() {
|
||||
final String[] args = new String[] {"arg1", "arg2", "arg3"};
|
||||
final String className = VarArgCredentialsProvider.class.getName();
|
||||
final String encodedValue = className + "|" + String.join("|", args);
|
||||
|
||||
final AwsCredentialsProvider provider = decoder.decodeValue(encodedValue);
|
||||
assertEquals(Arrays.toString(args), provider.resolveCredentials().accessKeyId());
|
||||
}
|
||||
|
||||
/**
|
||||
* This credentials provider will always succeed
|
||||
*/
|
||||
public static class AlwaysSucceedCredentialsProvider implements AwsCredentialsProvider {
|
||||
@Override
|
||||
public AwsCredentials resolveCredentials() {
|
||||
return AwsBasicCredentials.create(TEST_ACCESS_KEY_ID, TEST_SECRET_KEY);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This credentials provider needs a constructor call to instantiate it
|
||||
*/
|
||||
public static class ConstructorCredentialsProvider implements AwsCredentialsProvider {
|
||||
|
||||
private String arg1;
|
||||
private String arg2;
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
public ConstructorCredentialsProvider(String arg1) {
|
||||
this(arg1, "blank");
|
||||
}
|
||||
|
||||
public ConstructorCredentialsProvider(String arg1, String arg2) {
|
||||
this.arg1 = arg1;
|
||||
this.arg2 = arg2;
|
||||
}
|
||||
|
||||
@Override
|
||||
public AwsCredentials resolveCredentials() {
|
||||
return AwsBasicCredentials.create(arg1, arg2);
|
||||
}
|
||||
}
|
||||
|
||||
private static class VarArgCredentialsProvider implements AwsCredentialsProvider {
|
||||
|
||||
private final String[] args;
|
||||
|
||||
public VarArgCredentialsProvider(final String[] args) {
|
||||
this.args = args;
|
||||
}
|
||||
|
||||
@Override
|
||||
public AwsCredentials resolveCredentials() {
|
||||
// KISS solution to surface the constructor args
|
||||
final String flattenedArgs = Arrays.toString(args);
|
||||
return AwsBasicCredentials.create(flattenedArgs, flattenedArgs);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Credentials provider to test AWS SDK v2 create() methods for providers like ProfileCredentialsProvider
|
||||
*/
|
||||
public static class CreateProvider implements AwsCredentialsProvider {
|
||||
private String accessKeyId;
|
||||
|
||||
private CreateProvider(String accessKeyId) {
|
||||
this.accessKeyId = accessKeyId;
|
||||
}
|
||||
|
||||
public static CreateProvider create() {
|
||||
return new CreateProvider(TEST_ACCESS_KEY_ID);
|
||||
}
|
||||
|
||||
public static CreateProvider create(String accessKeyId) {
|
||||
return new CreateProvider(accessKeyId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public AwsCredentials resolveCredentials() {
|
||||
return AwsBasicCredentials.create(accessKeyId, TEST_SECRET_KEY);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -15,16 +15,14 @@
|
|||
|
||||
package software.amazon.kinesis.multilang.config;
|
||||
|
||||
import static org.hamcrest.CoreMatchers.containsString;
|
||||
import static org.hamcrest.CoreMatchers.equalTo;
|
||||
import static org.hamcrest.CoreMatchers.instanceOf;
|
||||
import static org.hamcrest.CoreMatchers.nullValue;
|
||||
import static org.junit.Assert.assertThat;
|
||||
import static org.junit.Assert.fail;
|
||||
|
||||
import java.util.function.Consumer;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
import lombok.Builder;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.ToString;
|
||||
import lombok.experimental.Accessors;
|
||||
import org.apache.commons.beanutils.BeanUtilsBean;
|
||||
import org.apache.commons.beanutils.ConvertUtilsBean;
|
||||
import org.junit.Before;
|
||||
|
|
@ -32,11 +30,12 @@ import org.junit.Rule;
|
|||
import org.junit.Test;
|
||||
import org.junit.rules.ExpectedException;
|
||||
|
||||
import lombok.Builder;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.ToString;
|
||||
import lombok.experimental.Accessors;
|
||||
import static org.hamcrest.CoreMatchers.containsString;
|
||||
import static org.hamcrest.CoreMatchers.equalTo;
|
||||
import static org.hamcrest.CoreMatchers.instanceOf;
|
||||
import static org.hamcrest.CoreMatchers.nullValue;
|
||||
import static org.junit.Assert.assertThat;
|
||||
import static org.junit.Assert.fail;
|
||||
|
||||
public class BuilderDynaBeanTest {
|
||||
|
||||
|
|
@ -109,8 +108,8 @@ public class BuilderDynaBeanTest {
|
|||
|
||||
@Test
|
||||
public void testComplexCreateAllParameters() throws Exception {
|
||||
TestComplexCreate expected = TestComplexCreate.create("real",
|
||||
TestSimpleBuilder.builder().stringL1("l1").longVal(10L).build());
|
||||
TestComplexCreate expected = TestComplexCreate.create(
|
||||
"real", TestSimpleBuilder.builder().stringL1("l1").longVal(10L).build());
|
||||
|
||||
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreate.class, convertUtilsBean);
|
||||
utilsBean.setProperty(builderDynaBean, "[0]", expected.realName);
|
||||
|
|
@ -136,8 +135,8 @@ public class BuilderDynaBeanTest {
|
|||
|
||||
@Test
|
||||
public void testComplexCreateComplexParameterOnly() throws Exception {
|
||||
TestComplexCreate expected = TestComplexCreate.create(null,
|
||||
TestSimpleBuilder.builder().stringL1("l1").longVal(10L).build());
|
||||
TestComplexCreate expected = TestComplexCreate.create(
|
||||
null, TestSimpleBuilder.builder().stringL1("l1").longVal(10L).build());
|
||||
|
||||
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreate.class, convertUtilsBean);
|
||||
utilsBean.setProperty(builderDynaBean, "[1].stringL1", expected.test1.stringL1);
|
||||
|
|
@ -161,7 +160,8 @@ public class BuilderDynaBeanTest {
|
|||
|
||||
@Test
|
||||
public void testSimpleBuilderAllParameters() throws Exception {
|
||||
TestSimpleBuilder expected = TestSimpleBuilder.builder().stringL1("l1").longVal(10L).build();
|
||||
TestSimpleBuilder expected =
|
||||
TestSimpleBuilder.builder().stringL1("l1").longVal(10L).build();
|
||||
|
||||
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSimpleBuilder.class, convertUtilsBean);
|
||||
utilsBean.setProperty(builderDynaBean, "stringL1", expected.stringL1);
|
||||
|
|
@ -213,12 +213,14 @@ public class BuilderDynaBeanTest {
|
|||
|
||||
@Test
|
||||
public void testComplexCreateSimpleBuilderVariantAllParameters() throws Exception {
|
||||
TestSimpleBuilder variant = TestSimpleBuilder.builder().longVal(10L).stringL1("variant").build();
|
||||
TestSimpleBuilder variant =
|
||||
TestSimpleBuilder.builder().longVal(10L).stringL1("variant").build();
|
||||
TestComplexCreateVariance expected = TestComplexCreateVariance.create("simple-builder", variant);
|
||||
|
||||
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean);
|
||||
utilsBean.setProperty(builderDynaBean, "[0]", expected.varianceName);
|
||||
utilsBean.setProperty(builderDynaBean, "[1].class", expected.variant.getClass().getName());
|
||||
utilsBean.setProperty(
|
||||
builderDynaBean, "[1].class", expected.variant.getClass().getName());
|
||||
utilsBean.setProperty(builderDynaBean, "[1].longVal", variant.longVal);
|
||||
utilsBean.setProperty(builderDynaBean, "[1].stringL1", variant.stringL1);
|
||||
|
||||
|
|
@ -229,8 +231,11 @@ public class BuilderDynaBeanTest {
|
|||
|
||||
@Test
|
||||
public void testComplexCreateVariantBuilderAllParameters() throws Exception {
|
||||
TestVariantBuilder variant = TestVariantBuilder.builder().variantBuilderName("variant-build").intClass(20)
|
||||
.testEnum(TestEnum.Blue).build();
|
||||
TestVariantBuilder variant = TestVariantBuilder.builder()
|
||||
.variantBuilderName("variant-build")
|
||||
.intClass(20)
|
||||
.testEnum(TestEnum.Blue)
|
||||
.build();
|
||||
TestComplexCreateVariance expected = TestComplexCreateVariance.create("builder-variant", variant);
|
||||
|
||||
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean);
|
||||
|
|
@ -264,13 +269,16 @@ public class BuilderDynaBeanTest {
|
|||
|
||||
@Test
|
||||
public void testComplexCreateVariantBuilderAllParametersPrefixWithJoiner() throws Exception {
|
||||
TestVariantBuilder variant = TestVariantBuilder.builder().variantBuilderName("variant-build").intClass(20)
|
||||
.testEnum(TestEnum.Blue).build();
|
||||
TestVariantBuilder variant = TestVariantBuilder.builder()
|
||||
.variantBuilderName("variant-build")
|
||||
.intClass(20)
|
||||
.testEnum(TestEnum.Blue)
|
||||
.build();
|
||||
TestComplexCreateVariance expected = TestComplexCreateVariance.create("builder-variant-prefix", variant);
|
||||
|
||||
String prefix = variant.getClass().getEnclosingClass().getName() + "$";
|
||||
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean,
|
||||
prefix);
|
||||
BuilderDynaBean builderDynaBean =
|
||||
new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean, prefix);
|
||||
utilsBean.setProperty(builderDynaBean, "[0]", expected.varianceName);
|
||||
utilsBean.setProperty(builderDynaBean, "[1].class", variant.getClass().getSimpleName());
|
||||
utilsBean.setProperty(builderDynaBean, "[1].variantBuilderName", variant.variantBuilderName);
|
||||
|
|
@ -284,13 +292,16 @@ public class BuilderDynaBeanTest {
|
|||
|
||||
@Test
|
||||
public void testComplexCreateVariantBuilderAllParametersPrefixWithOutJoiner() throws Exception {
|
||||
TestVariantBuilder variant = TestVariantBuilder.builder().variantBuilderName("variant-build").intClass(20)
|
||||
.testEnum(TestEnum.Blue).build();
|
||||
TestVariantBuilder variant = TestVariantBuilder.builder()
|
||||
.variantBuilderName("variant-build")
|
||||
.intClass(20)
|
||||
.testEnum(TestEnum.Blue)
|
||||
.build();
|
||||
TestComplexCreateVariance expected = TestComplexCreateVariance.create("builder-variant-prefix", variant);
|
||||
|
||||
String prefix = variant.getClass().getEnclosingClass().getName();
|
||||
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean,
|
||||
prefix);
|
||||
BuilderDynaBean builderDynaBean =
|
||||
new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean, prefix);
|
||||
utilsBean.setProperty(builderDynaBean, "[0]", expected.varianceName);
|
||||
utilsBean.setProperty(builderDynaBean, "[1].class", variant.getClass().getSimpleName());
|
||||
utilsBean.setProperty(builderDynaBean, "[1].variantBuilderName", variant.variantBuilderName);
|
||||
|
|
@ -330,11 +341,21 @@ public class BuilderDynaBeanTest {
|
|||
|
||||
@Test
|
||||
public void testComplexRootAllParameters() throws Exception {
|
||||
TestSimpleBuilder simpleBuilder = TestSimpleBuilder.builder().stringL1("simple-l1").longVal(20L).build();
|
||||
TestRootClass expected = TestRootClass.builder().intVal(10).stringVal("root").testEnum(TestEnum.Red)
|
||||
.testComplexCreate(TestComplexCreate.create("real",
|
||||
TestSimpleBuilder.builder().stringL1("complex-l1").longVal(10L).build()))
|
||||
.testSimpleBuilder(simpleBuilder).testSimpleCreate(TestSimpleCreate.create("first", "last")).build();
|
||||
TestSimpleBuilder simpleBuilder =
|
||||
TestSimpleBuilder.builder().stringL1("simple-l1").longVal(20L).build();
|
||||
TestRootClass expected = TestRootClass.builder()
|
||||
.intVal(10)
|
||||
.stringVal("root")
|
||||
.testEnum(TestEnum.Red)
|
||||
.testComplexCreate(TestComplexCreate.create(
|
||||
"real",
|
||||
TestSimpleBuilder.builder()
|
||||
.stringL1("complex-l1")
|
||||
.longVal(10L)
|
||||
.build()))
|
||||
.testSimpleBuilder(simpleBuilder)
|
||||
.testSimpleCreate(TestSimpleCreate.create("first", "last"))
|
||||
.build();
|
||||
|
||||
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestRootClass.class, convertUtilsBean);
|
||||
|
||||
|
|
@ -342,10 +363,10 @@ public class BuilderDynaBeanTest {
|
|||
utilsBean.setProperty(builderDynaBean, "stringVal", expected.stringVal);
|
||||
utilsBean.setProperty(builderDynaBean, "testEnum", expected.testEnum);
|
||||
utilsBean.setProperty(builderDynaBean, "testComplexCreate.[0]", expected.testComplexCreate.realName);
|
||||
utilsBean.setProperty(builderDynaBean, "testComplexCreate.[1].stringL1",
|
||||
expected.testComplexCreate.test1.stringL1);
|
||||
utilsBean.setProperty(builderDynaBean, "testComplexCreate.[1].longVal",
|
||||
expected.testComplexCreate.test1.longVal);
|
||||
utilsBean.setProperty(
|
||||
builderDynaBean, "testComplexCreate.[1].stringL1", expected.testComplexCreate.test1.stringL1);
|
||||
utilsBean.setProperty(
|
||||
builderDynaBean, "testComplexCreate.[1].longVal", expected.testComplexCreate.test1.longVal);
|
||||
utilsBean.setProperty(builderDynaBean, "testSimpleBuilder.class", TestSimpleBuilder.class.getName());
|
||||
utilsBean.setProperty(builderDynaBean, "testSimpleBuilder.stringL1", simpleBuilder.stringL1);
|
||||
utilsBean.setProperty(builderDynaBean, "testSimpleBuilder.longVal", simpleBuilder.longVal);
|
||||
|
|
@ -370,7 +391,11 @@ public class BuilderDynaBeanTest {
|
|||
|
||||
@Test
|
||||
public void testComplexRootTopLevelOnly() throws Exception {
|
||||
TestRootClass expected = TestRootClass.builder().intVal(10).stringVal("root").testEnum(TestEnum.Red).build();
|
||||
TestRootClass expected = TestRootClass.builder()
|
||||
.intVal(10)
|
||||
.stringVal("root")
|
||||
.testEnum(TestEnum.Red)
|
||||
.build();
|
||||
|
||||
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestRootClass.class, convertUtilsBean);
|
||||
|
||||
|
|
@ -385,12 +410,17 @@ public class BuilderDynaBeanTest {
|
|||
|
||||
@Test
|
||||
public void testSupplierNotUsed() throws Exception {
|
||||
TestVariantBuilder variant = TestVariantBuilder.builder().testEnum(TestEnum.Green).intClass(10)
|
||||
.variantBuilderName("variant-supplier").build();
|
||||
TestSupplierClass expected = TestSupplierClass.builder().variantClass(variant).build();
|
||||
TestVariantBuilder variant = TestVariantBuilder.builder()
|
||||
.testEnum(TestEnum.Green)
|
||||
.intClass(10)
|
||||
.variantBuilderName("variant-supplier")
|
||||
.build();
|
||||
TestSupplierClass expected =
|
||||
TestSupplierClass.builder().variantClass(variant).build();
|
||||
|
||||
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSupplierClass.class, convertUtilsBean);
|
||||
utilsBean.setProperty(builderDynaBean, "variantClass.class", variant.getClass().getName());
|
||||
utilsBean.setProperty(
|
||||
builderDynaBean, "variantClass.class", variant.getClass().getName());
|
||||
utilsBean.setProperty(builderDynaBean, "variantClass.testEnum", variant.testEnum);
|
||||
utilsBean.setProperty(builderDynaBean, "variantClass.intClass", variant.intClass);
|
||||
utilsBean.setProperty(builderDynaBean, "variantClass.variantBuilderName", variant.variantBuilderName);
|
||||
|
|
@ -422,8 +452,11 @@ public class BuilderDynaBeanTest {
|
|||
|
||||
@Test
|
||||
public void testVariantBuildsToSuperType() throws Exception {
|
||||
TestVariantBuilder expected = TestVariantBuilder.builder().intClass(10).testEnum(TestEnum.Green)
|
||||
.variantBuilderName("variant-super").build();
|
||||
TestVariantBuilder expected = TestVariantBuilder.builder()
|
||||
.intClass(10)
|
||||
.testEnum(TestEnum.Green)
|
||||
.variantBuilderName("variant-super")
|
||||
.build();
|
||||
|
||||
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestInterface.class, convertUtilsBean);
|
||||
utilsBean.setProperty(builderDynaBean, "class", expected.getClass().getName());
|
||||
|
|
@ -439,9 +472,11 @@ public class BuilderDynaBeanTest {
|
|||
@Test
|
||||
public void testEmptyPropertyHandler() throws Exception {
|
||||
String emptyPropertyValue = "test-property";
|
||||
TestVariantCreate expected = TestVariantCreate.create(emptyPropertyValue, (long) emptyPropertyValue.length(),
|
||||
emptyPropertyValue + "-vary");
|
||||
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestInterface.class, convertUtilsBean,
|
||||
TestVariantCreate expected = TestVariantCreate.create(
|
||||
emptyPropertyValue, (long) emptyPropertyValue.length(), emptyPropertyValue + "-vary");
|
||||
BuilderDynaBean builderDynaBean = new BuilderDynaBean(
|
||||
TestInterface.class,
|
||||
convertUtilsBean,
|
||||
s -> TestVariantCreate.create(s, (long) s.length(), s + "-vary"));
|
||||
utilsBean.setProperty(builderDynaBean, "", emptyPropertyValue);
|
||||
|
||||
|
|
@ -455,8 +490,8 @@ public class BuilderDynaBeanTest {
|
|||
thrown.expect(IllegalStateException.class);
|
||||
thrown.expectMessage(containsString("When a property handler is resolved further properties may not be set."));
|
||||
|
||||
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestInterface.class, convertUtilsBean,
|
||||
s -> TestVariantCreate.create("test", 10, "test"));
|
||||
BuilderDynaBean builderDynaBean = new BuilderDynaBean(
|
||||
TestInterface.class, convertUtilsBean, s -> TestVariantCreate.create("test", 10, "test"));
|
||||
utilsBean.setProperty(builderDynaBean, "", "test");
|
||||
utilsBean.setProperty(builderDynaBean, "[0]", "test");
|
||||
}
|
||||
|
|
@ -468,8 +503,8 @@ public class BuilderDynaBeanTest {
|
|||
thrown.expectMessage(containsString(TestInterface.class.getName()));
|
||||
thrown.expectMessage(containsString("cannot be assigned to"));
|
||||
|
||||
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestInterface.class, convertUtilsBean,
|
||||
s -> TestEnum.Green);
|
||||
BuilderDynaBean builderDynaBean =
|
||||
new BuilderDynaBean(TestInterface.class, convertUtilsBean, s -> TestEnum.Green);
|
||||
|
||||
utilsBean.setProperty(builderDynaBean, "", "test");
|
||||
|
||||
|
|
@ -478,8 +513,11 @@ public class BuilderDynaBeanTest {
|
|||
|
||||
@Test
|
||||
public void testSimpleArrayValues() throws Exception {
|
||||
SimpleArrayClassVariant expected = SimpleArrayClassVariant.builder().ints(new Integer[] { 1, 2, 3 })
|
||||
.variantName("simple-array").longs(new Long[] { 1L, 2L, 3L }).strings(new String[] { "a", "b", "c" })
|
||||
SimpleArrayClassVariant expected = SimpleArrayClassVariant.builder()
|
||||
.ints(new Integer[] {1, 2, 3})
|
||||
.variantName("simple-array")
|
||||
.longs(new Long[] {1L, 2L, 3L})
|
||||
.strings(new String[] {"a", "b", "c"})
|
||||
.build();
|
||||
|
||||
BuilderDynaBean builderDynaBean = new BuilderDynaBean(SimpleArrayClassVariant.class, convertUtilsBean);
|
||||
|
|
@ -503,12 +541,20 @@ public class BuilderDynaBeanTest {
|
|||
|
||||
@Test
|
||||
public void testComplexArrayValuesBuilder() throws Exception {
|
||||
TestVariantBuilder variant1 = TestVariantBuilder.builder().variantBuilderName("variant-1")
|
||||
.testEnum(TestEnum.Green).intClass(10).build();
|
||||
TestVariantBuilder variant2 = TestVariantBuilder.builder().variantBuilderName("variant-2")
|
||||
.testEnum(TestEnum.Blue).intClass(20).build();
|
||||
ComplexArrayClassVariant expected = ComplexArrayClassVariant.builder().variantName("complex-test")
|
||||
.tests(new TestInterface[] { variant1, variant2 }).build();
|
||||
TestVariantBuilder variant1 = TestVariantBuilder.builder()
|
||||
.variantBuilderName("variant-1")
|
||||
.testEnum(TestEnum.Green)
|
||||
.intClass(10)
|
||||
.build();
|
||||
TestVariantBuilder variant2 = TestVariantBuilder.builder()
|
||||
.variantBuilderName("variant-2")
|
||||
.testEnum(TestEnum.Blue)
|
||||
.intClass(20)
|
||||
.build();
|
||||
ComplexArrayClassVariant expected = ComplexArrayClassVariant.builder()
|
||||
.variantName("complex-test")
|
||||
.tests(new TestInterface[] {variant1, variant2})
|
||||
.build();
|
||||
|
||||
BuilderDynaBean builderDynaBean = new BuilderDynaBean(ComplexArrayClassVariant.class, convertUtilsBean);
|
||||
|
||||
|
|
@ -533,18 +579,22 @@ public class BuilderDynaBeanTest {
|
|||
TestVariantCreate variant1 = TestVariantCreate.create("variant-1", 10L, "vary-1");
|
||||
TestVariantCreate variant2 = TestVariantCreate.create("variant-2", 20L, "vary-2");
|
||||
|
||||
ComplexArrayClassVariant expected = ComplexArrayClassVariant.builder().variantName("create-test")
|
||||
.tests(new TestInterface[] { variant1, variant2 }).build();
|
||||
ComplexArrayClassVariant expected = ComplexArrayClassVariant.builder()
|
||||
.variantName("create-test")
|
||||
.tests(new TestInterface[] {variant1, variant2})
|
||||
.build();
|
||||
|
||||
BuilderDynaBean builderDynaBean = new BuilderDynaBean(ComplexArrayClassVariant.class, convertUtilsBean);
|
||||
|
||||
utilsBean.setProperty(builderDynaBean, "variantName", expected.variantName);
|
||||
utilsBean.setProperty(builderDynaBean, "tests[0].class", variant1.getClass().getName());
|
||||
utilsBean.setProperty(
|
||||
builderDynaBean, "tests[0].class", variant1.getClass().getName());
|
||||
utilsBean.setProperty(builderDynaBean, "tests[0].[0]", variant1.variantCreateName);
|
||||
utilsBean.setProperty(builderDynaBean, "tests[0].[1]", variant1.longClass);
|
||||
utilsBean.setProperty(builderDynaBean, "tests[0].[2]", variant1.varyString);
|
||||
|
||||
utilsBean.setProperty(builderDynaBean, "tests[1].class", variant2.getClass().getName());
|
||||
utilsBean.setProperty(
|
||||
builderDynaBean, "tests[1].class", variant2.getClass().getName());
|
||||
utilsBean.setProperty(builderDynaBean, "tests[1].[0]", variant2.variantCreateName);
|
||||
utilsBean.setProperty(builderDynaBean, "tests[1].[1]", variant2.longClass);
|
||||
utilsBean.setProperty(builderDynaBean, "tests[1].[2]", variant2.varyString);
|
||||
|
|
@ -552,7 +602,6 @@ public class BuilderDynaBeanTest {
|
|||
ComplexArrayClassVariant actual = builderDynaBean.build(ComplexArrayClassVariant.class);
|
||||
|
||||
assertThat(actual, equalTo(expected));
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -562,13 +611,18 @@ public class BuilderDynaBeanTest {
|
|||
if (i % 2 == 0) {
|
||||
variants[i] = TestVariantCreate.create("create-variant-" + i, i + 5, "vary-" + i);
|
||||
} else {
|
||||
variants[i] = TestVariantBuilder.builder().testEnum(TestEnum.values()[i % TestEnum.values().length])
|
||||
.intClass(i).variantBuilderName("builder-variant-" + i).build();
|
||||
variants[i] = TestVariantBuilder.builder()
|
||||
.testEnum(TestEnum.values()[i % TestEnum.values().length])
|
||||
.intClass(i)
|
||||
.variantBuilderName("builder-variant-" + i)
|
||||
.build();
|
||||
}
|
||||
}
|
||||
|
||||
ComplexArrayClassVariant expected = ComplexArrayClassVariant.builder().variantName("large-complex")
|
||||
.tests(variants).build();
|
||||
ComplexArrayClassVariant expected = ComplexArrayClassVariant.builder()
|
||||
.variantName("large-complex")
|
||||
.tests(variants)
|
||||
.build();
|
||||
|
||||
BuilderDynaBean builderDynaBean = new BuilderDynaBean(ComplexArrayClassVariant.class, convertUtilsBean);
|
||||
|
||||
|
|
@ -578,13 +632,15 @@ public class BuilderDynaBeanTest {
|
|||
TestInterface variant = variants[i];
|
||||
if (variant instanceof TestVariantCreate) {
|
||||
TestVariantCreate create = (TestVariantCreate) variant;
|
||||
utilsBean.setProperty(builderDynaBean, prefix + "class", create.getClass().getName());
|
||||
utilsBean.setProperty(
|
||||
builderDynaBean, prefix + "class", create.getClass().getName());
|
||||
utilsBean.setProperty(builderDynaBean, prefix + "[0]", create.variantCreateName);
|
||||
utilsBean.setProperty(builderDynaBean, prefix + "[1]", create.longClass);
|
||||
utilsBean.setProperty(builderDynaBean, prefix + "[2]", create.varyString);
|
||||
} else if (variant instanceof TestVariantBuilder) {
|
||||
TestVariantBuilder builder = (TestVariantBuilder) variant;
|
||||
utilsBean.setProperty(builderDynaBean, prefix + "class", builder.getClass().getName());
|
||||
utilsBean.setProperty(
|
||||
builderDynaBean, prefix + "class", builder.getClass().getName());
|
||||
utilsBean.setProperty(builderDynaBean, prefix + "variantBuilderName", builder.variantBuilderName);
|
||||
utilsBean.setProperty(builderDynaBean, prefix + "intClass", builder.intClass);
|
||||
utilsBean.setProperty(builderDynaBean, prefix + "testEnum", builder.testEnum);
|
||||
|
|
@ -667,25 +723,27 @@ public class BuilderDynaBeanTest {
|
|||
|
||||
@Test
|
||||
public void testAdditionalMutators() throws Exception {
|
||||
TestSimpleBuilder expected = TestSimpleBuilder.builder().stringL1("test").longVal(10L).build();
|
||||
TestSimpleBuilder expected =
|
||||
TestSimpleBuilder.builder().stringL1("test").longVal(10L).build();
|
||||
|
||||
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSimpleBuilder.class, convertUtilsBean);
|
||||
|
||||
utilsBean.setProperty(builderDynaBean, "stringL1", expected.stringL1);
|
||||
|
||||
TestSimpleBuilder actual = builderDynaBean.build(TestSimpleBuilder.class,
|
||||
b -> ((TestSimpleBuilder.TestSimpleBuilderBuilder) b).longVal(expected.longVal));
|
||||
TestSimpleBuilder actual =
|
||||
builderDynaBean.build(TestSimpleBuilder.class, b -> ((TestSimpleBuilder.TestSimpleBuilderBuilder) b)
|
||||
.longVal(expected.longVal));
|
||||
|
||||
assertThat(actual, equalTo(expected));
|
||||
}
|
||||
|
||||
public enum TestEnum {
|
||||
Red, Green, Blue
|
||||
Red,
|
||||
Green,
|
||||
Blue
|
||||
}
|
||||
|
||||
public interface TestInterface {
|
||||
|
||||
}
|
||||
public interface TestInterface {}
|
||||
|
||||
@Accessors(fluent = true)
|
||||
@ToString
|
||||
|
|
@ -838,7 +896,5 @@ public class BuilderDynaBeanTest {
|
|||
}
|
||||
|
||||
public String name = "default";
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -15,18 +15,17 @@
|
|||
|
||||
package software.amazon.kinesis.multilang.config;
|
||||
|
||||
import static org.hamcrest.CoreMatchers.equalTo;
|
||||
import static org.junit.Assert.assertThat;
|
||||
|
||||
import java.util.Optional;
|
||||
|
||||
import org.junit.Test;
|
||||
|
||||
import lombok.Builder;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import lombok.experimental.Accessors;
|
||||
import org.junit.Test;
|
||||
|
||||
import static org.hamcrest.CoreMatchers.equalTo;
|
||||
import static org.junit.Assert.assertThat;
|
||||
|
||||
public class ConfigurationSettableUtilsTest {
|
||||
|
||||
|
|
@ -44,7 +43,20 @@ public class ConfigurationSettableUtilsTest {
|
|||
public void testPrimitivesSet() {
|
||||
ConfigResult expected = ConfigResult.builder().rawInt(10).rawLong(15L).build();
|
||||
|
||||
ConfigObject configObject = ConfigObject.builder().rawInt(expected.rawInt).rawLong(expected.rawLong).build();
|
||||
ConfigObject configObject = ConfigObject.builder()
|
||||
.rawInt(expected.rawInt)
|
||||
.rawLong(expected.rawLong)
|
||||
.build();
|
||||
ConfigResult actual = resolve(configObject);
|
||||
|
||||
assertThat(actual, equalTo(expected));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBoolean() {
|
||||
ConfigResult expected = ConfigResult.builder().bool(false).build();
|
||||
|
||||
ConfigObject configObject = ConfigObject.builder().bool(expected.bool).build();
|
||||
ConfigResult actual = resolve(configObject);
|
||||
|
||||
assertThat(actual, equalTo(expected));
|
||||
|
|
@ -52,10 +64,14 @@ public class ConfigurationSettableUtilsTest {
|
|||
|
||||
@Test
|
||||
public void testHeapValuesSet() {
|
||||
ConfigResult expected = ConfigResult.builder().name("test").boxedInt(10).boxedLong(15L).build();
|
||||
ConfigResult expected =
|
||||
ConfigResult.builder().name("test").boxedInt(10).boxedLong(15L).build();
|
||||
|
||||
ConfigObject configObject = ConfigObject.builder().name(expected.name).boxedInt(expected.boxedInt.intValue())
|
||||
.boxedLong(expected.boxedLong.longValue()).build();
|
||||
ConfigObject configObject = ConfigObject.builder()
|
||||
.name(expected.name)
|
||||
.boxedInt(expected.boxedInt.intValue())
|
||||
.boxedLong(expected.boxedLong.longValue())
|
||||
.build();
|
||||
ConfigResult actual = resolve(configObject);
|
||||
|
||||
assertThat(actual, equalTo(expected));
|
||||
|
|
@ -63,27 +79,39 @@ public class ConfigurationSettableUtilsTest {
|
|||
|
||||
@Test
|
||||
public void testComplexValuesSet() {
|
||||
ComplexValue complexValue = ComplexValue.builder().name("complex").value(10).build();
|
||||
ConfigResult expected = ConfigResult.builder().complexValue(complexValue).build();
|
||||
ComplexValue complexValue =
|
||||
ComplexValue.builder().name("complex").value(10).build();
|
||||
ConfigResult expected =
|
||||
ConfigResult.builder().complexValue(complexValue).build();
|
||||
|
||||
ConfigObject configObject = ConfigObject.builder()
|
||||
.complexValue(ComplexValue.builder().name(complexValue.name).value(complexValue.value).build()).build();
|
||||
.complexValue(ComplexValue.builder()
|
||||
.name(complexValue.name)
|
||||
.value(complexValue.value)
|
||||
.build())
|
||||
.build();
|
||||
ConfigResult actual = resolve(configObject);
|
||||
|
||||
assertThat(actual, equalTo(expected));
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testOptionalValuesSet() {
|
||||
ComplexValue complexValue = ComplexValue.builder().name("optional-complex").value(20).build();
|
||||
ConfigResult expected = ConfigResult.builder().optionalString(Optional.of("test"))
|
||||
.optionalInteger(Optional.of(10)).optionalLong(Optional.of(15L))
|
||||
.optionalComplexValue(Optional.of(complexValue)).build();
|
||||
ComplexValue complexValue =
|
||||
ComplexValue.builder().name("optional-complex").value(20).build();
|
||||
ConfigResult expected = ConfigResult.builder()
|
||||
.optionalString(Optional.of("test"))
|
||||
.optionalInteger(Optional.of(10))
|
||||
.optionalLong(Optional.of(15L))
|
||||
.optionalComplexValue(Optional.of(complexValue))
|
||||
.build();
|
||||
|
||||
ConfigObject configObject = ConfigObject.builder().optionalString(expected.optionalString.get())
|
||||
.optionalInteger(expected.optionalInteger.get()).optionalLong(expected.optionalLong.get())
|
||||
.optionalComplexValue(expected.optionalComplexValue.get()).build();
|
||||
ConfigObject configObject = ConfigObject.builder()
|
||||
.optionalString(expected.optionalString.get())
|
||||
.optionalInteger(expected.optionalInteger.get())
|
||||
.optionalLong(expected.optionalLong.get())
|
||||
.optionalComplexValue(expected.optionalComplexValue.get())
|
||||
.build();
|
||||
ConfigResult actual = resolve(configObject);
|
||||
|
||||
assertThat(actual, equalTo(expected));
|
||||
|
|
@ -91,20 +119,29 @@ public class ConfigurationSettableUtilsTest {
|
|||
|
||||
@Test
|
||||
public void testRenamedRawValues() {
|
||||
ComplexValue complexValue = ComplexValue.builder().name("renamed-complex").value(20).build();
|
||||
ConfigResult expected = ConfigResult.builder().renamedString("renamed").renamedInt(10)
|
||||
.renamedOptionalString(Optional.of("renamed-optional")).renamedComplexValue(complexValue).build();
|
||||
ComplexValue complexValue =
|
||||
ComplexValue.builder().name("renamed-complex").value(20).build();
|
||||
ConfigResult expected = ConfigResult.builder()
|
||||
.renamedString("renamed")
|
||||
.renamedInt(10)
|
||||
.renamedOptionalString(Optional.of("renamed-optional"))
|
||||
.renamedComplexValue(complexValue)
|
||||
.build();
|
||||
|
||||
ConfigObject configObject = ConfigObject.builder().toRenameString(expected.renamedString)
|
||||
.toRenameInt(expected.renamedInt).toRenameComplexValue(complexValue)
|
||||
.optionalToRename(expected.renamedOptionalString.get()).build();
|
||||
ConfigObject configObject = ConfigObject.builder()
|
||||
.toRenameString(expected.renamedString)
|
||||
.toRenameInt(expected.renamedInt)
|
||||
.toRenameComplexValue(complexValue)
|
||||
.optionalToRename(expected.renamedOptionalString.get())
|
||||
.build();
|
||||
ConfigResult actual = resolve(configObject);
|
||||
|
||||
assertThat(actual, equalTo(expected));
|
||||
}
|
||||
|
||||
private ConfigResult resolve(ConfigObject configObject) {
|
||||
return ConfigurationSettableUtils.resolveFields(configObject, ConfigResult.builder().build());
|
||||
return ConfigurationSettableUtils.resolveFields(
|
||||
configObject, ConfigResult.builder().build());
|
||||
}
|
||||
|
||||
@Accessors(fluent = true)
|
||||
|
|
@ -120,6 +157,9 @@ public class ConfigurationSettableUtilsTest {
|
|||
private Long boxedLong;
|
||||
private ComplexValue complexValue;
|
||||
|
||||
@Builder.Default
|
||||
private Boolean bool = true;
|
||||
|
||||
private Optional<String> optionalString;
|
||||
private Optional<Integer> optionalInteger;
|
||||
private Optional<Long> optionalLong;
|
||||
|
|
@ -129,7 +169,6 @@ public class ConfigurationSettableUtilsTest {
|
|||
private int renamedInt;
|
||||
private Optional<String> renamedOptionalString;
|
||||
private ComplexValue renamedComplexValue;
|
||||
|
||||
}
|
||||
|
||||
@Accessors(fluent = true)
|
||||
|
|
@ -145,35 +184,51 @@ public class ConfigurationSettableUtilsTest {
|
|||
|
||||
@ConfigurationSettable(configurationClass = ConfigResult.class)
|
||||
private String name;
|
||||
|
||||
@ConfigurationSettable(configurationClass = ConfigResult.class)
|
||||
private int rawInt;
|
||||
|
||||
@ConfigurationSettable(configurationClass = ConfigResult.class)
|
||||
@Builder.Default
|
||||
private Boolean bool = true;
|
||||
|
||||
@ConfigurationSettable(configurationClass = ConfigResult.class)
|
||||
private Integer boxedInt;
|
||||
|
||||
@ConfigurationSettable(configurationClass = ConfigResult.class)
|
||||
private long rawLong;
|
||||
|
||||
@ConfigurationSettable(configurationClass = ConfigResult.class)
|
||||
private Long boxedLong;
|
||||
|
||||
@ConfigurationSettable(configurationClass = ConfigResult.class)
|
||||
private ComplexValue complexValue;
|
||||
|
||||
@ConfigurationSettable(configurationClass = ConfigResult.class, convertToOptional = true)
|
||||
private String optionalString;
|
||||
|
||||
@ConfigurationSettable(configurationClass = ConfigResult.class, convertToOptional = true)
|
||||
private Integer optionalInteger;
|
||||
|
||||
@ConfigurationSettable(configurationClass = ConfigResult.class, convertToOptional = true)
|
||||
private Long optionalLong;
|
||||
|
||||
@ConfigurationSettable(configurationClass = ConfigResult.class, convertToOptional = true)
|
||||
private ComplexValue optionalComplexValue;
|
||||
|
||||
@ConfigurationSettable(configurationClass = ConfigResult.class, methodName = "renamedString")
|
||||
private String toRenameString;
|
||||
|
||||
@ConfigurationSettable(configurationClass = ConfigResult.class, methodName = "renamedInt")
|
||||
private int toRenameInt;
|
||||
@ConfigurationSettable(configurationClass = ConfigResult.class, methodName = "renamedOptionalString", convertToOptional = true)
|
||||
|
||||
@ConfigurationSettable(
|
||||
configurationClass = ConfigResult.class,
|
||||
methodName = "renamedOptionalString",
|
||||
convertToOptional = true)
|
||||
private String optionalToRename;
|
||||
|
||||
@ConfigurationSettable(configurationClass = ConfigResult.class, methodName = "renamedComplexValue")
|
||||
private ComplexValue toRenameComplexValue;
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -14,12 +14,12 @@
|
|||
*/
|
||||
package software.amazon.kinesis.multilang.config;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
import java.util.Date;
|
||||
|
||||
import org.junit.Test;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
public class DatePropertyValueDecoderTest {
|
||||
|
||||
private DatePropertyValueDecoder decoder = new DatePropertyValueDecoder();
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ import org.junit.Test;
|
|||
import org.junit.runner.RunWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.runners.MockitoJUnitRunner;
|
||||
|
||||
import software.amazon.awssdk.services.kinesis.KinesisAsyncClient;
|
||||
import software.amazon.kinesis.retrieval.fanout.FanOutConfig;
|
||||
|
||||
|
|
@ -57,11 +56,15 @@ public class FanoutConfigBeanTest {
|
|||
assertThat(fanOutConfig.applicationName(), equalTo(configuration.getApplicationName()));
|
||||
assertThat(fanOutConfig.consumerArn(), equalTo(fanoutConfigBean.getConsumerArn()));
|
||||
assertThat(fanOutConfig.consumerName(), equalTo(fanoutConfigBean.getConsumerName()));
|
||||
assertThat(fanOutConfig.maxDescribeStreamConsumerRetries(), equalTo(fanoutConfigBean.getMaxDescribeStreamConsumerRetries()));
|
||||
assertThat(fanOutConfig.maxDescribeStreamSummaryRetries(), equalTo(fanoutConfigBean.getMaxDescribeStreamSummaryRetries()));
|
||||
assertThat(fanOutConfig.registerStreamConsumerRetries(), equalTo(fanoutConfigBean.getRegisterStreamConsumerRetries()));
|
||||
assertThat(
|
||||
fanOutConfig.maxDescribeStreamConsumerRetries(),
|
||||
equalTo(fanoutConfigBean.getMaxDescribeStreamConsumerRetries()));
|
||||
assertThat(
|
||||
fanOutConfig.maxDescribeStreamSummaryRetries(),
|
||||
equalTo(fanoutConfigBean.getMaxDescribeStreamSummaryRetries()));
|
||||
assertThat(
|
||||
fanOutConfig.registerStreamConsumerRetries(),
|
||||
equalTo(fanoutConfigBean.getRegisterStreamConsumerRetries()));
|
||||
assertThat(fanOutConfig.retryBackoffMillis(), equalTo(fanoutConfigBean.getRetryBackoffMillis()));
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -14,38 +14,39 @@
|
|||
*/
|
||||
package software.amazon.kinesis.multilang.config;
|
||||
|
||||
import static org.hamcrest.CoreMatchers.equalTo;
|
||||
import static org.hamcrest.CoreMatchers.nullValue;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
import static org.junit.Assert.assertThat;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.junit.Assert.fail;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.InputStream;
|
||||
import java.net.URI;
|
||||
import java.util.Arrays;
|
||||
import java.util.Date;
|
||||
import java.util.HashSet;
|
||||
import java.util.NoSuchElementException;
|
||||
import java.util.Set;
|
||||
|
||||
import com.amazonaws.auth.AWSCredentials;
|
||||
import com.amazonaws.auth.AWSCredentialsProvider;
|
||||
import com.amazonaws.auth.BasicAWSCredentials;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.commons.lang3.exception.ExceptionUtils;
|
||||
import org.junit.Test;
|
||||
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
|
||||
import org.junit.runner.RunWith;
|
||||
import org.mockito.runners.MockitoJUnitRunner;
|
||||
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
|
||||
import software.amazon.awssdk.auth.credentials.AwsCredentials;
|
||||
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
|
||||
import software.amazon.awssdk.services.dynamodb.model.BillingMode;
|
||||
import software.amazon.kinesis.common.InitialPositionInStream;
|
||||
import software.amazon.kinesis.coordinator.CoordinatorConfig;
|
||||
import software.amazon.kinesis.metrics.MetricsLevel;
|
||||
|
||||
import static org.hamcrest.CoreMatchers.equalTo;
|
||||
import static org.hamcrest.CoreMatchers.nullValue;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
import static org.junit.Assert.assertNull;
|
||||
import static org.junit.Assert.assertThat;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.junit.Assert.fail;
|
||||
|
||||
@RunWith(MockitoJUnitRunner.class)
|
||||
public class KinesisClientLibConfiguratorTest {
|
||||
|
||||
|
|
@ -58,20 +59,35 @@ public class KinesisClientLibConfiguratorTest {
|
|||
|
||||
@Test
|
||||
public void testWithBasicSetup() {
|
||||
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a",
|
||||
"applicationName = b", "AWSCredentialsProvider = " + credentialName1, "workerId = 123" }, '\n'));
|
||||
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
|
||||
new String[] {
|
||||
"streamName = a",
|
||||
"applicationName = b",
|
||||
"AwsCredentialsProvider = " + credentialName1,
|
||||
"workerId = 123"
|
||||
},
|
||||
'\n'));
|
||||
assertEquals(config.getApplicationName(), "b");
|
||||
assertEquals(config.getStreamName(), "a");
|
||||
assertEquals(config.getWorkerIdentifier(), "123");
|
||||
assertThat(config.getMaxGetRecordsThreadPool(), nullValue());
|
||||
assertThat(config.getRetryGetRecordsInSeconds(), nullValue());
|
||||
assertNull(config.getGracefulLeaseHandoffTimeoutMillis());
|
||||
assertNull(config.getIsGracefulLeaseHandoffEnabled());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWithLongVariables() {
|
||||
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "applicationName = app",
|
||||
"streamName = 123", "AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
|
||||
"workerId = 123", "failoverTimeMillis = 100", "shardSyncIntervalMillis = 500" }, '\n'));
|
||||
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
|
||||
new String[] {
|
||||
"applicationName = app",
|
||||
"streamName = 123",
|
||||
"AwsCredentialsProvider = " + credentialName1 + ", " + credentialName2,
|
||||
"workerId = 123",
|
||||
"failoverTimeMillis = 100",
|
||||
"shardSyncIntervalMillis = 500"
|
||||
},
|
||||
'\n'));
|
||||
|
||||
assertEquals(config.getApplicationName(), "app");
|
||||
assertEquals(config.getStreamName(), "123");
|
||||
|
|
@ -83,9 +99,14 @@ public class KinesisClientLibConfiguratorTest {
|
|||
@Test
|
||||
public void testWithInitialPositionInStreamExtended() {
|
||||
long epochTimeInSeconds = 1617406032;
|
||||
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "applicationName = app",
|
||||
"streamName = 123", "AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
|
||||
"initialPositionInStreamExtended = " + epochTimeInSeconds}, '\n'));
|
||||
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
|
||||
new String[] {
|
||||
"applicationName = app",
|
||||
"streamName = 123",
|
||||
"AwsCredentialsProvider = " + credentialName1 + ", " + credentialName2,
|
||||
"initialPositionInStreamExtended = " + epochTimeInSeconds
|
||||
},
|
||||
'\n'));
|
||||
|
||||
assertEquals(config.getInitialPositionInStreamExtended().getTimestamp(), new Date(epochTimeInSeconds * 1000L));
|
||||
assertEquals(config.getInitialPositionInStream(), InitialPositionInStream.AT_TIMESTAMP);
|
||||
|
|
@ -96,9 +117,14 @@ public class KinesisClientLibConfiguratorTest {
|
|||
// AT_TIMESTAMP cannot be used as initialPositionInStream. If a user wants to specify AT_TIMESTAMP,
|
||||
// they must specify the time with initialPositionInStreamExtended.
|
||||
try {
|
||||
getConfiguration(StringUtils.join(new String[] { "applicationName = app",
|
||||
"streamName = 123", "AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
|
||||
"initialPositionInStream = AT_TIMESTAMP"}, '\n'));
|
||||
getConfiguration(StringUtils.join(
|
||||
new String[] {
|
||||
"applicationName = app",
|
||||
"streamName = 123",
|
||||
"AwsCredentialsProvider = " + credentialName1 + ", " + credentialName2,
|
||||
"initialPositionInStream = AT_TIMESTAMP"
|
||||
},
|
||||
'\n'));
|
||||
fail("Should have thrown when initialPositionInStream is set to AT_TIMESTAMP");
|
||||
} catch (Exception e) {
|
||||
Throwable rootCause = ExceptionUtils.getRootCause(e);
|
||||
|
|
@ -111,9 +137,14 @@ public class KinesisClientLibConfiguratorTest {
|
|||
// initialPositionInStreamExtended takes a long value indicating seconds since epoch. If a non-long
|
||||
// value is provided, the constructor should throw an IllegalArgumentException exception.
|
||||
try {
|
||||
getConfiguration(StringUtils.join(new String[] { "applicationName = app",
|
||||
"streamName = 123", "AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
|
||||
"initialPositionInStreamExtended = null"}, '\n'));
|
||||
getConfiguration(StringUtils.join(
|
||||
new String[] {
|
||||
"applicationName = app",
|
||||
"streamName = 123",
|
||||
"AwsCredentialsProvider = " + credentialName1 + ", " + credentialName2,
|
||||
"initialPositionInStreamExtended = null"
|
||||
},
|
||||
'\n'));
|
||||
fail("Should have thrown when initialPositionInStreamExtended is set to null");
|
||||
} catch (Exception e) {
|
||||
Throwable rootCause = ExceptionUtils.getRootCause(e);
|
||||
|
|
@ -121,11 +152,161 @@ public class KinesisClientLibConfiguratorTest {
|
|||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGracefulLeaseHandoffConfig() {
|
||||
final Long testGracefulLeaseHandoffTimeoutMillis = 12345L;
|
||||
final boolean testGracefulLeaseHandoffEnabled = true;
|
||||
|
||||
final MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
|
||||
new String[] {
|
||||
"applicationName = dummyApplicationName",
|
||||
"streamName = dummyStreamName",
|
||||
"AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
|
||||
"gracefulLeaseHandoffTimeoutMillis = " + testGracefulLeaseHandoffTimeoutMillis,
|
||||
"isGracefulLeaseHandoffEnabled = " + testGracefulLeaseHandoffEnabled
|
||||
},
|
||||
'\n'));
|
||||
|
||||
assertEquals(testGracefulLeaseHandoffTimeoutMillis, config.getGracefulLeaseHandoffTimeoutMillis());
|
||||
assertEquals(testGracefulLeaseHandoffEnabled, config.getIsGracefulLeaseHandoffEnabled());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testClientVersionConfig() {
|
||||
final CoordinatorConfig.ClientVersionConfig testClientVersionConfig = Arrays.stream(
|
||||
CoordinatorConfig.ClientVersionConfig.values())
|
||||
.findAny()
|
||||
.orElseThrow(NoSuchElementException::new);
|
||||
|
||||
final MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
|
||||
new String[] {
|
||||
"applicationName = dummyApplicationName",
|
||||
"streamName = dummyStreamName",
|
||||
"AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
|
||||
"clientVersionConfig = " + testClientVersionConfig.name()
|
||||
},
|
||||
'\n'));
|
||||
|
||||
assertEquals(testClientVersionConfig, config.getClientVersionConfig());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCoordinatorStateConfig() {
|
||||
final String testCoordinatorStateTableName = "CoordState";
|
||||
final BillingMode testCoordinatorStateBillingMode = BillingMode.PAY_PER_REQUEST;
|
||||
final long testCoordinatorStateReadCapacity = 123;
|
||||
final long testCoordinatorStateWriteCapacity = 123;
|
||||
|
||||
final MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
|
||||
new String[] {
|
||||
"applicationName = dummyApplicationName",
|
||||
"streamName = dummyStreamName",
|
||||
"AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
|
||||
"coordinatorStateTableName = " + testCoordinatorStateTableName,
|
||||
"coordinatorStateBillingMode = " + testCoordinatorStateBillingMode.name(),
|
||||
"coordinatorStateReadCapacity = " + testCoordinatorStateReadCapacity,
|
||||
"coordinatorStateWriteCapacity = " + testCoordinatorStateWriteCapacity
|
||||
},
|
||||
'\n'));
|
||||
|
||||
assertEquals(testCoordinatorStateTableName, config.getCoordinatorStateTableName());
|
||||
assertEquals(testCoordinatorStateBillingMode, config.getCoordinatorStateBillingMode());
|
||||
assertEquals(testCoordinatorStateReadCapacity, config.getCoordinatorStateReadCapacity());
|
||||
assertEquals(testCoordinatorStateWriteCapacity, config.getCoordinatorStateWriteCapacity());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWorkerUtilizationAwareAssignmentConfig() {
|
||||
final long testInMemoryWorkerMetricsCaptureFrequencyMillis = 123;
|
||||
final long testWorkerMetricsReporterFreqInMillis = 123;
|
||||
final long testNoOfPersistedMetricsPerWorkerMetrics = 123;
|
||||
final Boolean testDisableWorkerMetrics = true;
|
||||
final double testMaxThroughputPerHostKBps = 123;
|
||||
final long testDampeningPercentage = 12;
|
||||
final long testReBalanceThresholdPercentage = 12;
|
||||
final Boolean testAllowThroughputOvershoot = false;
|
||||
final long testVarianceBalancingFrequency = 12;
|
||||
final double testWorkerMetricsEMAAlpha = .123;
|
||||
|
||||
final MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
|
||||
new String[] {
|
||||
"applicationName = dummyApplicationName",
|
||||
"streamName = dummyStreamName",
|
||||
"AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
|
||||
"inMemoryWorkerMetricsCaptureFrequencyMillis = " + testInMemoryWorkerMetricsCaptureFrequencyMillis,
|
||||
"workerMetricsReporterFreqInMillis = " + testWorkerMetricsReporterFreqInMillis,
|
||||
"noOfPersistedMetricsPerWorkerMetrics = " + testNoOfPersistedMetricsPerWorkerMetrics,
|
||||
"disableWorkerMetrics = " + testDisableWorkerMetrics,
|
||||
"maxThroughputPerHostKBps = " + testMaxThroughputPerHostKBps,
|
||||
"dampeningPercentage = " + testDampeningPercentage,
|
||||
"reBalanceThresholdPercentage = " + testReBalanceThresholdPercentage,
|
||||
"allowThroughputOvershoot = " + testAllowThroughputOvershoot,
|
||||
"varianceBalancingFrequency = " + testVarianceBalancingFrequency,
|
||||
"workerMetricsEMAAlpha = " + testWorkerMetricsEMAAlpha
|
||||
},
|
||||
'\n'));
|
||||
|
||||
assertEquals(
|
||||
testInMemoryWorkerMetricsCaptureFrequencyMillis,
|
||||
config.getInMemoryWorkerMetricsCaptureFrequencyMillis());
|
||||
assertEquals(testWorkerMetricsReporterFreqInMillis, config.getWorkerMetricsReporterFreqInMillis());
|
||||
assertEquals(testNoOfPersistedMetricsPerWorkerMetrics, config.getNoOfPersistedMetricsPerWorkerMetrics());
|
||||
assertEquals(testDisableWorkerMetrics, config.getDisableWorkerMetrics());
|
||||
assertEquals(testMaxThroughputPerHostKBps, config.getMaxThroughputPerHostKBps(), 0.0001);
|
||||
assertEquals(testDampeningPercentage, config.getDampeningPercentage());
|
||||
assertEquals(testReBalanceThresholdPercentage, config.getReBalanceThresholdPercentage());
|
||||
assertEquals(testAllowThroughputOvershoot, config.getAllowThroughputOvershoot());
|
||||
assertEquals(testVarianceBalancingFrequency, config.getVarianceBalancingFrequency());
|
||||
assertEquals(testWorkerMetricsEMAAlpha, config.getWorkerMetricsEMAAlpha(), 0.0001);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWorkerMetricsConfig() {
|
||||
final String testWorkerMetricsTableName = "CoordState";
|
||||
final BillingMode testWorkerMetricsBillingMode = BillingMode.PROVISIONED;
|
||||
final long testWorkerMetricsReadCapacity = 123;
|
||||
final long testWorkerMetricsWriteCapacity = 123;
|
||||
|
||||
final MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
|
||||
new String[] {
|
||||
"applicationName = dummyApplicationName",
|
||||
"streamName = dummyStreamName",
|
||||
"AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
|
||||
"workerMetricsTableName = " + testWorkerMetricsTableName,
|
||||
"workerMetricsBillingMode = " + testWorkerMetricsBillingMode.name(),
|
||||
"workerMetricsReadCapacity = " + testWorkerMetricsReadCapacity,
|
||||
"workerMetricsWriteCapacity = " + testWorkerMetricsWriteCapacity
|
||||
},
|
||||
'\n'));
|
||||
|
||||
assertEquals(testWorkerMetricsTableName, config.getWorkerMetricsTableName());
|
||||
assertEquals(testWorkerMetricsBillingMode, config.getWorkerMetricsBillingMode());
|
||||
assertEquals(testWorkerMetricsReadCapacity, config.getWorkerMetricsReadCapacity());
|
||||
assertEquals(testWorkerMetricsWriteCapacity, config.getWorkerMetricsWriteCapacity());
|
||||
}
|
||||
|
||||
@Test(expected = IllegalArgumentException.class)
|
||||
public void testInvalidClientVersionConfig() {
|
||||
getConfiguration(StringUtils.join(
|
||||
new String[] {
|
||||
"applicationName = dummyApplicationName",
|
||||
"streamName = dummyStreamName",
|
||||
"AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
|
||||
"clientVersionConfig = " + "invalid_client_version_config"
|
||||
},
|
||||
'\n'));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWithUnsupportedClientConfigurationVariables() {
|
||||
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
|
||||
new String[] { "AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2, "workerId = id",
|
||||
"kinesisClientConfig = {}", "streamName = stream", "applicationName = b" },
|
||||
new String[] {
|
||||
"AwsCredentialsProvider = " + credentialName1 + ", " + credentialName2,
|
||||
"workerId = id",
|
||||
"kinesisClientConfig = {}",
|
||||
"streamName = stream",
|
||||
"applicationName = b"
|
||||
},
|
||||
'\n'));
|
||||
|
||||
assertEquals(config.getApplicationName(), "b");
|
||||
|
|
@ -136,10 +317,18 @@ public class KinesisClientLibConfiguratorTest {
|
|||
|
||||
@Test
|
||||
public void testWithIntVariables() {
|
||||
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = kinesis",
|
||||
"AWSCredentialsProvider = " + credentialName2 + ", " + credentialName1, "workerId = w123",
|
||||
"maxRecords = 10", "metricsMaxQueueSize = 20", "applicationName = kinesis",
|
||||
"retryGetRecordsInSeconds = 2", "maxGetRecordsThreadPool = 1" }, '\n'));
|
||||
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
|
||||
new String[] {
|
||||
"streamName = kinesis",
|
||||
"AwsCredentialsProvider = " + credentialName2 + ", " + credentialName1,
|
||||
"workerId = w123",
|
||||
"maxRecords = 10",
|
||||
"metricsMaxQueueSize = 20",
|
||||
"applicationName = kinesis",
|
||||
"retryGetRecordsInSeconds = 2",
|
||||
"maxGetRecordsThreadPool = 1"
|
||||
},
|
||||
'\n'));
|
||||
|
||||
assertEquals(config.getApplicationName(), "kinesis");
|
||||
assertEquals(config.getStreamName(), "kinesis");
|
||||
|
|
@ -152,9 +341,15 @@ public class KinesisClientLibConfiguratorTest {
|
|||
|
||||
@Test
|
||||
public void testWithBooleanVariables() {
|
||||
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a",
|
||||
"applicationName = b", "AWSCredentialsProvider = ABCD, " + credentialName1, "workerId = 0",
|
||||
"cleanupLeasesUponShardCompletion = false", "validateSequenceNumberBeforeCheckpointing = true" },
|
||||
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
|
||||
new String[] {
|
||||
"streamName = a",
|
||||
"applicationName = b",
|
||||
"AwsCredentialsProvider = ABCD, " + credentialName1,
|
||||
"workerId = 0",
|
||||
"cleanupLeasesUponShardCompletion = false",
|
||||
"validateSequenceNumberBeforeCheckpointing = true"
|
||||
},
|
||||
'\n'));
|
||||
|
||||
assertEquals(config.getApplicationName(), "b");
|
||||
|
|
@ -166,9 +361,16 @@ public class KinesisClientLibConfiguratorTest {
|
|||
|
||||
@Test
|
||||
public void testWithStringVariables() {
|
||||
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a",
|
||||
"applicationName = b", "AWSCredentialsProvider = ABCD," + credentialName1, "workerId = 1",
|
||||
"kinesisEndpoint = https://kinesis", "metricsLevel = SUMMARY" }, '\n'));
|
||||
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
|
||||
new String[] {
|
||||
"streamName = a",
|
||||
"applicationName = b",
|
||||
"AwsCredentialsProvider = ABCD," + credentialName1,
|
||||
"workerId = 1",
|
||||
"kinesisEndpoint = https://kinesis",
|
||||
"metricsLevel = SUMMARY"
|
||||
},
|
||||
'\n'));
|
||||
|
||||
assertEquals(config.getWorkerIdentifier(), "1");
|
||||
assertEquals(config.getKinesisClient().get("endpointOverride"), URI.create("https://kinesis"));
|
||||
|
|
@ -177,38 +379,66 @@ public class KinesisClientLibConfiguratorTest {
|
|||
|
||||
@Test
|
||||
public void testWithSetVariables() {
|
||||
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a",
|
||||
"applicationName = b", "AWSCredentialsProvider = ABCD," + credentialName1, "workerId = 1",
|
||||
"metricsEnabledDimensions = ShardId, WorkerIdentifier" }, '\n'));
|
||||
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
|
||||
new String[] {
|
||||
"streamName = a",
|
||||
"applicationName = b",
|
||||
"AwsCredentialsProvider = ABCD," + credentialName1,
|
||||
"workerId = 1",
|
||||
"metricsEnabledDimensions = ShardId, WorkerIdentifier"
|
||||
},
|
||||
'\n'));
|
||||
|
||||
Set<String> expectedMetricsEnabledDimensions = ImmutableSet.<String>builder()
|
||||
.add("ShardId", "WorkerIdentifier").build();
|
||||
assertThat(new HashSet<>(Arrays.asList(config.getMetricsEnabledDimensions())), equalTo(expectedMetricsEnabledDimensions));
|
||||
.add("ShardId", "WorkerIdentifier")
|
||||
.build();
|
||||
assertThat(
|
||||
new HashSet<>(Arrays.asList(config.getMetricsEnabledDimensions())),
|
||||
equalTo(expectedMetricsEnabledDimensions));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWithInitialPositionInStreamTrimHorizon() {
|
||||
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a",
|
||||
"applicationName = b", "AWSCredentialsProvider = ABCD," + credentialName1, "workerId = 123",
|
||||
"initialPositionInStream = TriM_Horizon" }, '\n'));
|
||||
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
|
||||
new String[] {
|
||||
"streamName = a",
|
||||
"applicationName = b",
|
||||
"AwsCredentialsProvider = ABCD," + credentialName1,
|
||||
"workerId = 123",
|
||||
"initialPositionInStream = TriM_Horizon"
|
||||
},
|
||||
'\n'));
|
||||
|
||||
assertEquals(config.getInitialPositionInStream(), InitialPositionInStream.TRIM_HORIZON);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWithInitialPositionInStreamLatest() {
|
||||
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a",
|
||||
"applicationName = b", "AWSCredentialsProvider = ABCD," + credentialName1, "workerId = 123",
|
||||
"initialPositionInStream = LateSt" }, '\n'));
|
||||
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
|
||||
new String[] {
|
||||
"streamName = a",
|
||||
"applicationName = b",
|
||||
"AwsCredentialsProvider = ABCD," + credentialName1,
|
||||
"workerId = 123",
|
||||
"initialPositionInStream = LateSt"
|
||||
},
|
||||
'\n'));
|
||||
|
||||
assertEquals(config.getInitialPositionInStream(), InitialPositionInStream.LATEST);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSkippingNonKCLVariables() {
|
||||
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a",
|
||||
"applicationName = b", "AWSCredentialsProvider = ABCD," + credentialName1, "workerId = 123",
|
||||
"initialPositionInStream = TriM_Horizon", "abc = 1" }, '\n'));
|
||||
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
|
||||
new String[] {
|
||||
"streamName = a",
|
||||
"applicationName = b",
|
||||
"AwsCredentialsProvider = ABCD," + credentialName1,
|
||||
"workerId = 123",
|
||||
"initialPositionInStream = TriM_Horizon",
|
||||
"abc = 1"
|
||||
},
|
||||
'\n'));
|
||||
|
||||
assertEquals(config.getApplicationName(), "b");
|
||||
assertEquals(config.getStreamName(), "a");
|
||||
|
|
@ -218,33 +448,61 @@ public class KinesisClientLibConfiguratorTest {
|
|||
|
||||
@Test
|
||||
public void testEmptyOptionalVariables() {
|
||||
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a",
|
||||
"applicationName = b", "AWSCredentialsProvider = ABCD," + credentialName1, "workerId = 123",
|
||||
"initialPositionInStream = TriM_Horizon", "maxGetRecordsThreadPool = 1" }, '\n'));
|
||||
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
|
||||
new String[] {
|
||||
"streamName = a",
|
||||
"applicationName = b",
|
||||
"AwsCredentialsProvider = ABCD," + credentialName1,
|
||||
"workerId = 123",
|
||||
"initialPositionInStream = TriM_Horizon",
|
||||
"maxGetRecordsThreadPool = 1"
|
||||
},
|
||||
'\n'));
|
||||
assertThat(config.getMaxGetRecordsThreadPool(), equalTo(1));
|
||||
assertThat(config.getRetryGetRecordsInSeconds(), nullValue());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWithZeroValue() {
|
||||
String test = StringUtils.join(new String[] { "streamName = a", "applicationName = b",
|
||||
"AWSCredentialsProvider = ABCD," + credentialName1, "workerId = 123",
|
||||
"initialPositionInStream = TriM_Horizon", "maxGetRecordsThreadPool = 0",
|
||||
"retryGetRecordsInSeconds = 0" }, '\n');
|
||||
String test = StringUtils.join(
|
||||
new String[] {
|
||||
"streamName = a",
|
||||
"applicationName = b",
|
||||
"AwsCredentialsProvider = ABCD," + credentialName1,
|
||||
"workerId = 123",
|
||||
"initialPositionInStream = TriM_Horizon",
|
||||
"maxGetRecordsThreadPool = 0",
|
||||
"retryGetRecordsInSeconds = 0"
|
||||
},
|
||||
'\n');
|
||||
getConfiguration(test);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWithInvalidIntValue() {
|
||||
String test = StringUtils.join(new String[] { "streamName = a", "applicationName = b",
|
||||
"AWSCredentialsProvider = " + credentialName1, "workerId = 123", "failoverTimeMillis = 100nf" }, '\n');
|
||||
String test = StringUtils.join(
|
||||
new String[] {
|
||||
"streamName = a",
|
||||
"applicationName = b",
|
||||
"AwsCredentialsProvider = " + credentialName1,
|
||||
"workerId = 123",
|
||||
"failoverTimeMillis = 100nf"
|
||||
},
|
||||
'\n');
|
||||
getConfiguration(test);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWithNegativeIntValue() {
|
||||
String test = StringUtils.join(new String[] { "streamName = a", "applicationName = b",
|
||||
"AWSCredentialsProvider = " + credentialName1, "workerId = 123", "failoverTimeMillis = -12" }, '\n');
|
||||
String test = StringUtils.join(
|
||||
new String[] {
|
||||
"streamName = a",
|
||||
"applicationName = b",
|
||||
"AwsCredentialsProvider = " + credentialName1,
|
||||
"workerId = 123",
|
||||
"failoverTimeMillis = -12"
|
||||
},
|
||||
'\n');
|
||||
|
||||
// separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement
|
||||
getConfiguration(test);
|
||||
|
|
@ -252,8 +510,15 @@ public class KinesisClientLibConfiguratorTest {
|
|||
|
||||
@Test(expected = IllegalArgumentException.class)
|
||||
public void testWithMissingCredentialsProvider() {
|
||||
String test = StringUtils.join(new String[] { "streamName = a", "applicationName = b", "workerId = 123",
|
||||
"failoverTimeMillis = 100", "shardSyncIntervalMillis = 500" }, '\n');
|
||||
String test = StringUtils.join(
|
||||
new String[] {
|
||||
"streamName = a",
|
||||
"applicationName = b",
|
||||
"workerId = 123",
|
||||
"failoverTimeMillis = 100",
|
||||
"shardSyncIntervalMillis = 500"
|
||||
},
|
||||
'\n');
|
||||
|
||||
// separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement
|
||||
getConfiguration(test);
|
||||
|
|
@ -262,8 +527,13 @@ public class KinesisClientLibConfiguratorTest {
|
|||
@Test
|
||||
public void testWithMissingWorkerId() {
|
||||
String test = StringUtils.join(
|
||||
new String[] { "streamName = a", "applicationName = b", "AWSCredentialsProvider = " + credentialName1,
|
||||
"failoverTimeMillis = 100", "shardSyncIntervalMillis = 500" },
|
||||
new String[] {
|
||||
"streamName = a",
|
||||
"applicationName = b",
|
||||
"AwsCredentialsProvider = " + credentialName1,
|
||||
"failoverTimeMillis = 100",
|
||||
"shardSyncIntervalMillis = 500"
|
||||
},
|
||||
'\n');
|
||||
MultiLangDaemonConfiguration config = getConfiguration(test);
|
||||
|
||||
|
|
@ -274,88 +544,157 @@ public class KinesisClientLibConfiguratorTest {
|
|||
|
||||
@Test(expected = NullPointerException.class)
|
||||
public void testWithMissingStreamNameAndMissingStreamArn() {
|
||||
String test = StringUtils.join(new String[] {
|
||||
String test = StringUtils.join(
|
||||
new String[] {
|
||||
"applicationName = b",
|
||||
"AWSCredentialsProvider = " + credentialName1,
|
||||
"AwsCredentialsProvider = " + credentialName1,
|
||||
"workerId = 123",
|
||||
"failoverTimeMillis = 100" },
|
||||
"failoverTimeMillis = 100"
|
||||
},
|
||||
'\n');
|
||||
getConfiguration(test);
|
||||
}
|
||||
|
||||
@Test(expected = IllegalArgumentException.class)
|
||||
public void testWithEmptyStreamNameAndMissingStreamArn() {
|
||||
String test = StringUtils.join(new String[] {
|
||||
String test = StringUtils.join(
|
||||
new String[] {
|
||||
"applicationName = b",
|
||||
"AWSCredentialsProvider = " + credentialName1,
|
||||
"AwsCredentialsProvider = " + credentialName1,
|
||||
"workerId = 123",
|
||||
"failoverTimeMillis = 100",
|
||||
"streamName = ",
|
||||
"streamArn = "},
|
||||
"streamArn = "
|
||||
},
|
||||
'\n');
|
||||
getConfiguration(test);
|
||||
}
|
||||
|
||||
@Test(expected = NullPointerException.class)
|
||||
public void testWithMissingApplicationName() {
|
||||
String test = StringUtils.join(new String[] { "streamName = a", "AWSCredentialsProvider = " + credentialName1,
|
||||
"workerId = 123", "failoverTimeMillis = 100" }, '\n');
|
||||
String test = StringUtils.join(
|
||||
new String[] {
|
||||
"streamName = a",
|
||||
"AwsCredentialsProvider = " + credentialName1,
|
||||
"workerId = 123",
|
||||
"failoverTimeMillis = 100"
|
||||
},
|
||||
'\n');
|
||||
getConfiguration(test);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWithAWSCredentialsFailed() {
|
||||
public void testWithAwsCredentialsFailed() {
|
||||
String test = StringUtils.join(
|
||||
new String[] { "streamName = a", "applicationName = b", "AWSCredentialsProvider = " + credentialName2,
|
||||
"failoverTimeMillis = 100", "shardSyncIntervalMillis = 500" },
|
||||
new String[] {
|
||||
"streamName = a",
|
||||
"applicationName = b",
|
||||
"AwsCredentialsProvider = " + credentialName2,
|
||||
"failoverTimeMillis = 100",
|
||||
"shardSyncIntervalMillis = 500"
|
||||
},
|
||||
'\n');
|
||||
MultiLangDaemonConfiguration config = getConfiguration(test);
|
||||
|
||||
// separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement
|
||||
try {
|
||||
config.getKinesisCredentialsProvider().build(AwsCredentialsProvider.class).resolveCredentials();
|
||||
config.getKinesisCredentialsProvider()
|
||||
.build(AwsCredentialsProvider.class)
|
||||
.resolveCredentials();
|
||||
fail("expect failure with wrong credentials provider");
|
||||
} catch (Exception e) {
|
||||
// succeed
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: fix this test
|
||||
@Test
|
||||
public void testWithDifferentAWSCredentialsForDynamoDBAndCloudWatch() {
|
||||
String test = StringUtils.join(new String[] { "streamName = a", "applicationName = b",
|
||||
"AWSCredentialsProvider = " + credentialNameKinesis,
|
||||
"AWSCredentialsProviderDynamoDB = " + credentialNameDynamoDB,
|
||||
"AWSCredentialsProviderCloudWatch = " + credentialNameCloudWatch, "failoverTimeMillis = 100",
|
||||
"shardSyncIntervalMillis = 500" }, '\n');
|
||||
public void testProcessKeyWithExpectedCasing() {
|
||||
String key = "AwsCredentialsProvider";
|
||||
String result = configurator.processKey(key);
|
||||
assertEquals("awsCredentialsProvider", result);
|
||||
}
|
||||
|
||||
// separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement
|
||||
final MultiLangDaemonConfiguration config = getConfiguration(test);
|
||||
config.getKinesisCredentialsProvider().build(AwsCredentialsProvider.class).resolveCredentials();
|
||||
config.getDynamoDBCredentialsProvider().build(AwsCredentialsProvider.class).resolveCredentials();
|
||||
config.getCloudWatchCredentialsProvider().build(AwsCredentialsProvider.class).resolveCredentials();
|
||||
@Test
|
||||
public void testProcessKeyWithOldCasing() {
|
||||
String key = "AWSCredentialsProvider";
|
||||
String result = configurator.processKey(key);
|
||||
assertEquals("awsCredentialsProvider", result);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testProcessKeyWithMixedCasing() {
|
||||
String key = "AwScReDeNtIaLsPrOvIdEr";
|
||||
String result = configurator.processKey(key);
|
||||
assertEquals("awsCredentialsProvider", result);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testProcessKeyWithSuffix() {
|
||||
String key = "awscredentialsproviderDynamoDB";
|
||||
String result = configurator.processKey(key);
|
||||
assertEquals("awsCredentialsProviderDynamoDB", result);
|
||||
}
|
||||
|
||||
// TODO: fix this test
|
||||
@Test
|
||||
public void testWithDifferentAWSCredentialsForDynamoDBAndCloudWatchFailed() {
|
||||
String test = StringUtils.join(new String[] { "streamName = a", "applicationName = b",
|
||||
"AWSCredentialsProvider = " + credentialNameKinesis,
|
||||
"AWSCredentialsProviderDynamoDB = " + credentialName2,
|
||||
"AWSCredentialsProviderCloudWatch = " + credentialName2, "failoverTimeMillis = 100",
|
||||
"shardSyncIntervalMillis = 500" }, '\n');
|
||||
public void testWithDifferentAwsCredentialsForDynamoDBAndCloudWatch() {
|
||||
String test = StringUtils.join(
|
||||
new String[] {
|
||||
"streamName = a",
|
||||
"applicationName = b",
|
||||
"AwsCredentialsProvider = " + credentialNameKinesis,
|
||||
"AwsCredentialsProviderDynamoDB = " + credentialNameDynamoDB,
|
||||
"AwsCredentialsProviderCloudWatch = " + credentialNameCloudWatch,
|
||||
"failoverTimeMillis = 100",
|
||||
"shardSyncIntervalMillis = 500"
|
||||
},
|
||||
'\n');
|
||||
|
||||
// separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement
|
||||
final MultiLangDaemonConfiguration config = getConfiguration(test);
|
||||
config.getKinesisCredentialsProvider().build(AwsCredentialsProvider.class).resolveCredentials();
|
||||
config.getKinesisCredentialsProvider()
|
||||
.build(AwsCredentialsProvider.class)
|
||||
.resolveCredentials();
|
||||
config.getDynamoDBCredentialsProvider()
|
||||
.build(AwsCredentialsProvider.class)
|
||||
.resolveCredentials();
|
||||
config.getCloudWatchCredentialsProvider()
|
||||
.build(AwsCredentialsProvider.class)
|
||||
.resolveCredentials();
|
||||
}
|
||||
|
||||
// TODO: fix this test
|
||||
@Test
|
||||
public void testWithDifferentAwsCredentialsForDynamoDBAndCloudWatchFailed() {
|
||||
String test = StringUtils.join(
|
||||
new String[] {
|
||||
"streamName = a",
|
||||
"applicationName = b",
|
||||
"AwsCredentialsProvider = " + credentialNameKinesis,
|
||||
"AwsCredentialsProviderDynamoDB = " + credentialName2,
|
||||
"AwsCredentialsProviderCloudWatch = " + credentialName2,
|
||||
"failoverTimeMillis = 100",
|
||||
"shardSyncIntervalMillis = 500"
|
||||
},
|
||||
'\n');
|
||||
|
||||
// separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement
|
||||
final MultiLangDaemonConfiguration config = getConfiguration(test);
|
||||
config.getKinesisCredentialsProvider()
|
||||
.build(AwsCredentialsProvider.class)
|
||||
.resolveCredentials();
|
||||
try {
|
||||
config.getDynamoDBCredentialsProvider().build(AwsCredentialsProvider.class).resolveCredentials();
|
||||
config.getDynamoDBCredentialsProvider()
|
||||
.build(AwsCredentialsProvider.class)
|
||||
.resolveCredentials();
|
||||
fail("DynamoDB credential providers should fail.");
|
||||
} catch (Exception e) {
|
||||
// succeed
|
||||
}
|
||||
try {
|
||||
config.getCloudWatchCredentialsProvider().build(AwsCredentialsProvider.class).resolveCredentials();
|
||||
config.getCloudWatchCredentialsProvider()
|
||||
.build(AwsCredentialsProvider.class)
|
||||
.resolveCredentials();
|
||||
fail("CloudWatch credential providers should fail.");
|
||||
} catch (Exception e) {
|
||||
// succeed
|
||||
|
|
@ -365,81 +704,52 @@ public class KinesisClientLibConfiguratorTest {
|
|||
/**
|
||||
* This credentials provider will always succeed
|
||||
*/
|
||||
public static class AlwaysSucceedCredentialsProvider implements AWSCredentialsProvider {
|
||||
|
||||
public static class AlwaysSucceedCredentialsProvider implements AwsCredentialsProvider {
|
||||
@Override
|
||||
public AWSCredentials getCredentials() {
|
||||
return new BasicAWSCredentials("a", "b");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void refresh() {
|
||||
|
||||
public AwsCredentials resolveCredentials() {
|
||||
return AwsBasicCredentials.create("a", "b");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This credentials provider will always succeed
|
||||
*/
|
||||
public static class AlwaysSucceedCredentialsProviderKinesis implements AWSCredentialsProvider {
|
||||
|
||||
public static class AlwaysSucceedCredentialsProviderKinesis implements AwsCredentialsProvider {
|
||||
@Override
|
||||
public AWSCredentials getCredentials() {
|
||||
return new BasicAWSCredentials("", "");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void refresh() {
|
||||
|
||||
public AwsCredentials resolveCredentials() {
|
||||
return AwsBasicCredentials.create("DUMMY_ACCESS_KEY_ID", "DUMMY_SECRET_ACCESS_KEY");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This credentials provider will always succeed
|
||||
*/
|
||||
public static class AlwaysSucceedCredentialsProviderDynamoDB implements AWSCredentialsProvider {
|
||||
|
||||
public static class AlwaysSucceedCredentialsProviderDynamoDB implements AwsCredentialsProvider {
|
||||
@Override
|
||||
public AWSCredentials getCredentials() {
|
||||
return new BasicAWSCredentials("", "");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void refresh() {
|
||||
|
||||
public AwsCredentials resolveCredentials() {
|
||||
return AwsBasicCredentials.create("DUMMY_ACCESS_KEY_ID", "DUMMY_SECRET_ACCESS_KEY");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This credentials provider will always succeed
|
||||
*/
|
||||
public static class AlwaysSucceedCredentialsProviderCloudWatch implements AWSCredentialsProvider {
|
||||
|
||||
public static class AlwaysSucceedCredentialsProviderCloudWatch implements AwsCredentialsProvider {
|
||||
@Override
|
||||
public AWSCredentials getCredentials() {
|
||||
return new BasicAWSCredentials("", "");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void refresh() {
|
||||
|
||||
public AwsCredentials resolveCredentials() {
|
||||
return AwsBasicCredentials.create("DUMMY_ACCESS_KEY_ID", "DUMMY_SECRET_ACCESS_KEY");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This credentials provider will always fail
|
||||
*/
|
||||
public static class AlwaysFailCredentialsProvider implements AWSCredentialsProvider {
|
||||
public static class AlwaysFailCredentialsProvider implements AwsCredentialsProvider {
|
||||
|
||||
@Override
|
||||
public AWSCredentials getCredentials() {
|
||||
public AwsCredentials resolveCredentials() {
|
||||
throw new IllegalArgumentException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void refresh() {
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
private MultiLangDaemonConfiguration getConfiguration(String configString) {
|
||||
|
|
|
|||
|
|
@ -15,9 +15,8 @@
|
|||
|
||||
package software.amazon.kinesis.multilang.config;
|
||||
|
||||
import static org.hamcrest.CoreMatchers.equalTo;
|
||||
import static org.hamcrest.CoreMatchers.instanceOf;
|
||||
import static org.junit.Assert.assertThat;
|
||||
import java.util.Arrays;
|
||||
import java.util.NoSuchElementException;
|
||||
|
||||
import org.apache.commons.beanutils.BeanUtilsBean;
|
||||
import org.apache.commons.beanutils.ConvertUtilsBean;
|
||||
|
|
@ -28,17 +27,34 @@ import org.junit.Test;
|
|||
import org.junit.rules.ExpectedException;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Mockito;
|
||||
import org.mockito.runners.MockitoJUnitRunner;
|
||||
|
||||
import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
|
||||
import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient;
|
||||
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
|
||||
import software.amazon.awssdk.services.dynamodb.model.BillingMode;
|
||||
import software.amazon.awssdk.services.kinesis.KinesisAsyncClient;
|
||||
import software.amazon.kinesis.common.ConfigsBuilder;
|
||||
import software.amazon.kinesis.coordinator.CoordinatorConfig;
|
||||
import software.amazon.kinesis.leases.LeaseManagementConfig;
|
||||
import software.amazon.kinesis.processor.ShardRecordProcessorFactory;
|
||||
import software.amazon.kinesis.retrieval.fanout.FanOutConfig;
|
||||
import software.amazon.kinesis.retrieval.polling.PollingConfig;
|
||||
|
||||
import static org.hamcrest.CoreMatchers.equalTo;
|
||||
import static org.hamcrest.CoreMatchers.instanceOf;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertNotEquals;
|
||||
import static org.junit.Assert.assertThat;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
@RunWith(MockitoJUnitRunner.class)
|
||||
public class MultiLangDaemonConfigurationTest {
|
||||
|
||||
private static final String AWS_REGION_PROPERTY_NAME = "aws.region";
|
||||
private static final String DUMMY_APPLICATION_NAME = "dummyApplicationName";
|
||||
private static final String DUMMY_STREAM_NAME = "dummyStreamName";
|
||||
|
||||
private BeanUtilsBean utilsBean;
|
||||
private ConvertUtilsBean convertUtilsBean;
|
||||
|
|
@ -67,11 +83,10 @@ public class MultiLangDaemonConfigurationTest {
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
public MultiLangDaemonConfiguration baseConfiguration() {
|
||||
MultiLangDaemonConfiguration configuration = new MultiLangDaemonConfiguration(utilsBean, convertUtilsBean);
|
||||
configuration.setApplicationName("Test");
|
||||
configuration.setStreamName("Test");
|
||||
configuration.setApplicationName(DUMMY_APPLICATION_NAME);
|
||||
configuration.setStreamName(DUMMY_STREAM_NAME);
|
||||
configuration.getKinesisCredentialsProvider().set("class", DefaultCredentialsProvider.class.getName());
|
||||
|
||||
return configuration;
|
||||
|
|
@ -82,33 +97,289 @@ public class MultiLangDaemonConfigurationTest {
|
|||
MultiLangDaemonConfiguration configuration = baseConfiguration();
|
||||
configuration.setMaxLeasesForWorker(10);
|
||||
|
||||
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration
|
||||
.resolvedConfiguration(shardRecordProcessorFactory);
|
||||
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
|
||||
configuration.resolvedConfiguration(shardRecordProcessorFactory);
|
||||
|
||||
assertThat(resolvedConfiguration.leaseManagementConfig.maxLeasesForWorker(), equalTo(10));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSetEnablePriorityLeaseAssignment() {
|
||||
MultiLangDaemonConfiguration configuration = baseConfiguration();
|
||||
configuration.setEnablePriorityLeaseAssignment(false);
|
||||
|
||||
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
|
||||
configuration.resolvedConfiguration(shardRecordProcessorFactory);
|
||||
|
||||
assertThat(resolvedConfiguration.leaseManagementConfig.enablePriorityLeaseAssignment(), equalTo(false));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSetLeaseTableDeletionProtectionEnabledToTrue() {
|
||||
MultiLangDaemonConfiguration configuration = baseConfiguration();
|
||||
configuration.setLeaseTableDeletionProtectionEnabled(true);
|
||||
|
||||
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
|
||||
configuration.resolvedConfiguration(shardRecordProcessorFactory);
|
||||
|
||||
assertTrue(resolvedConfiguration.leaseManagementConfig.leaseTableDeletionProtectionEnabled());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGracefulLeaseHandoffConfig() {
|
||||
final LeaseManagementConfig.GracefulLeaseHandoffConfig defaultGracefulLeaseHandoffConfig =
|
||||
getTestConfigsBuilder().leaseManagementConfig().gracefulLeaseHandoffConfig();
|
||||
|
||||
final long testGracefulLeaseHandoffTimeoutMillis =
|
||||
defaultGracefulLeaseHandoffConfig.gracefulLeaseHandoffTimeoutMillis() + 12345;
|
||||
final boolean testGracefulLeaseHandoffEnabled =
|
||||
!defaultGracefulLeaseHandoffConfig.isGracefulLeaseHandoffEnabled();
|
||||
|
||||
final MultiLangDaemonConfiguration configuration = baseConfiguration();
|
||||
configuration.setGracefulLeaseHandoffTimeoutMillis(testGracefulLeaseHandoffTimeoutMillis);
|
||||
configuration.setIsGracefulLeaseHandoffEnabled(testGracefulLeaseHandoffEnabled);
|
||||
|
||||
final MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
|
||||
configuration.resolvedConfiguration(shardRecordProcessorFactory);
|
||||
|
||||
final LeaseManagementConfig.GracefulLeaseHandoffConfig gracefulLeaseHandoffConfig =
|
||||
resolvedConfiguration.leaseManagementConfig.gracefulLeaseHandoffConfig();
|
||||
|
||||
assertEquals(
|
||||
testGracefulLeaseHandoffTimeoutMillis, gracefulLeaseHandoffConfig.gracefulLeaseHandoffTimeoutMillis());
|
||||
assertEquals(testGracefulLeaseHandoffEnabled, gracefulLeaseHandoffConfig.isGracefulLeaseHandoffEnabled());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGracefulLeaseHandoffUsesDefaults() {
|
||||
final MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
|
||||
baseConfiguration().resolvedConfiguration(shardRecordProcessorFactory);
|
||||
|
||||
final LeaseManagementConfig.GracefulLeaseHandoffConfig gracefulLeaseHandoffConfig =
|
||||
resolvedConfiguration.leaseManagementConfig.gracefulLeaseHandoffConfig();
|
||||
|
||||
final LeaseManagementConfig.GracefulLeaseHandoffConfig defaultGracefulLeaseHandoffConfig =
|
||||
getTestConfigsBuilder().leaseManagementConfig().gracefulLeaseHandoffConfig();
|
||||
|
||||
assertEquals(defaultGracefulLeaseHandoffConfig, gracefulLeaseHandoffConfig);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWorkerUtilizationAwareAssignmentConfig() {
|
||||
MultiLangDaemonConfiguration configuration = baseConfiguration();
|
||||
|
||||
configuration.setInMemoryWorkerMetricsCaptureFrequencyMillis(123);
|
||||
configuration.setWorkerMetricsReporterFreqInMillis(123);
|
||||
configuration.setNoOfPersistedMetricsPerWorkerMetrics(123);
|
||||
configuration.setDisableWorkerMetrics(true);
|
||||
configuration.setMaxThroughputPerHostKBps(.123);
|
||||
configuration.setDampeningPercentage(12);
|
||||
configuration.setReBalanceThresholdPercentage(12);
|
||||
configuration.setAllowThroughputOvershoot(false);
|
||||
configuration.setVarianceBalancingFrequency(12);
|
||||
configuration.setWorkerMetricsEMAAlpha(.123);
|
||||
|
||||
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
|
||||
configuration.resolvedConfiguration(shardRecordProcessorFactory);
|
||||
LeaseManagementConfig leaseManagementConfig = resolvedConfiguration.leaseManagementConfig;
|
||||
LeaseManagementConfig.WorkerUtilizationAwareAssignmentConfig config =
|
||||
leaseManagementConfig.workerUtilizationAwareAssignmentConfig();
|
||||
|
||||
assertEquals(config.inMemoryWorkerMetricsCaptureFrequencyMillis(), 123);
|
||||
assertEquals(config.workerMetricsReporterFreqInMillis(), 123);
|
||||
assertEquals(config.noOfPersistedMetricsPerWorkerMetrics(), 123);
|
||||
assertTrue(config.disableWorkerMetrics());
|
||||
assertEquals(config.maxThroughputPerHostKBps(), .123, .25);
|
||||
assertEquals(config.dampeningPercentage(), 12);
|
||||
assertEquals(config.reBalanceThresholdPercentage(), 12);
|
||||
assertFalse(config.allowThroughputOvershoot());
|
||||
assertEquals(config.varianceBalancingFrequency(), 12);
|
||||
assertEquals(config.workerMetricsEMAAlpha(), .123, .25);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWorkerUtilizationAwareAssignmentConfigUsesDefaults() {
|
||||
final LeaseManagementConfig.WorkerUtilizationAwareAssignmentConfig defaultWorkerUtilAwareAssignmentConfig =
|
||||
getTestConfigsBuilder().leaseManagementConfig().workerUtilizationAwareAssignmentConfig();
|
||||
|
||||
final MultiLangDaemonConfiguration configuration = baseConfiguration();
|
||||
configuration.setVarianceBalancingFrequency(
|
||||
defaultWorkerUtilAwareAssignmentConfig.varianceBalancingFrequency() + 12345);
|
||||
|
||||
final MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
|
||||
configuration.resolvedConfiguration(shardRecordProcessorFactory);
|
||||
|
||||
final LeaseManagementConfig.WorkerUtilizationAwareAssignmentConfig resolvedWorkerUtilAwareAssignmentConfig =
|
||||
resolvedConfiguration.leaseManagementConfig.workerUtilizationAwareAssignmentConfig();
|
||||
|
||||
assertNotEquals(defaultWorkerUtilAwareAssignmentConfig, resolvedWorkerUtilAwareAssignmentConfig);
|
||||
|
||||
// apart from the single updated configuration, all other config values should be equal to the default
|
||||
resolvedWorkerUtilAwareAssignmentConfig.varianceBalancingFrequency(
|
||||
defaultWorkerUtilAwareAssignmentConfig.varianceBalancingFrequency());
|
||||
assertEquals(defaultWorkerUtilAwareAssignmentConfig, resolvedWorkerUtilAwareAssignmentConfig);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWorkerMetricsTableConfigBean() {
|
||||
final BillingMode testWorkerMetricsTableBillingMode = BillingMode.PROVISIONED;
|
||||
|
||||
MultiLangDaemonConfiguration configuration = baseConfiguration();
|
||||
|
||||
configuration.setWorkerMetricsTableName("testTable");
|
||||
configuration.setWorkerMetricsBillingMode(testWorkerMetricsTableBillingMode);
|
||||
configuration.setWorkerMetricsReadCapacity(123);
|
||||
configuration.setWorkerMetricsWriteCapacity(123);
|
||||
|
||||
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
|
||||
configuration.resolvedConfiguration(shardRecordProcessorFactory);
|
||||
LeaseManagementConfig leaseManagementConfig = resolvedConfiguration.leaseManagementConfig;
|
||||
LeaseManagementConfig.WorkerUtilizationAwareAssignmentConfig workerUtilizationConfig =
|
||||
leaseManagementConfig.workerUtilizationAwareAssignmentConfig();
|
||||
LeaseManagementConfig.WorkerMetricsTableConfig workerMetricsConfig =
|
||||
workerUtilizationConfig.workerMetricsTableConfig();
|
||||
|
||||
assertEquals(workerMetricsConfig.tableName(), "testTable");
|
||||
assertEquals(workerMetricsConfig.billingMode(), testWorkerMetricsTableBillingMode);
|
||||
assertEquals(workerMetricsConfig.readCapacity(), 123);
|
||||
assertEquals(workerMetricsConfig.writeCapacity(), 123);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWorkerMetricsTableConfigUsesDefaults() {
|
||||
final LeaseManagementConfig.WorkerMetricsTableConfig defaultWorkerMetricsTableConfig = getTestConfigsBuilder()
|
||||
.leaseManagementConfig()
|
||||
.workerUtilizationAwareAssignmentConfig()
|
||||
.workerMetricsTableConfig();
|
||||
|
||||
final MultiLangDaemonConfiguration configuration = baseConfiguration();
|
||||
configuration.setWorkerMetricsBillingMode(Arrays.stream(BillingMode.values())
|
||||
.filter(billingMode -> billingMode != defaultWorkerMetricsTableConfig.billingMode())
|
||||
.findFirst()
|
||||
.orElseThrow(NoSuchElementException::new));
|
||||
|
||||
final MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
|
||||
configuration.resolvedConfiguration(shardRecordProcessorFactory);
|
||||
|
||||
final LeaseManagementConfig.WorkerMetricsTableConfig resolvedWorkerMetricsTableConfig = resolvedConfiguration
|
||||
.leaseManagementConfig
|
||||
.workerUtilizationAwareAssignmentConfig()
|
||||
.workerMetricsTableConfig();
|
||||
|
||||
assertNotEquals(defaultWorkerMetricsTableConfig, resolvedWorkerMetricsTableConfig);
|
||||
|
||||
// apart from the single updated configuration, all other config values should be equal to the default
|
||||
resolvedWorkerMetricsTableConfig.billingMode(defaultWorkerMetricsTableConfig.billingMode());
|
||||
assertEquals(defaultWorkerMetricsTableConfig, resolvedWorkerMetricsTableConfig);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCoordinatorStateTableConfigBean() {
|
||||
final BillingMode testWorkerMetricsTableBillingMode = BillingMode.PAY_PER_REQUEST;
|
||||
|
||||
MultiLangDaemonConfiguration configuration = baseConfiguration();
|
||||
|
||||
configuration.setCoordinatorStateTableName("testTable");
|
||||
configuration.setCoordinatorStateBillingMode(testWorkerMetricsTableBillingMode);
|
||||
configuration.setCoordinatorStateReadCapacity(123);
|
||||
configuration.setCoordinatorStateWriteCapacity(123);
|
||||
|
||||
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
|
||||
configuration.resolvedConfiguration(shardRecordProcessorFactory);
|
||||
CoordinatorConfig coordinatorConfig = resolvedConfiguration.getCoordinatorConfig();
|
||||
CoordinatorConfig.CoordinatorStateTableConfig coordinatorStateConfig =
|
||||
coordinatorConfig.coordinatorStateTableConfig();
|
||||
assertEquals(coordinatorStateConfig.tableName(), "testTable");
|
||||
assertEquals(coordinatorStateConfig.billingMode(), testWorkerMetricsTableBillingMode);
|
||||
assertEquals(coordinatorStateConfig.readCapacity(), 123);
|
||||
assertEquals(coordinatorStateConfig.writeCapacity(), 123);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCoordinatorStateTableConfigUsesDefaults() {
|
||||
final CoordinatorConfig.CoordinatorStateTableConfig defaultCoordinatorStateTableConfig =
|
||||
getTestConfigsBuilder().coordinatorConfig().coordinatorStateTableConfig();
|
||||
|
||||
final MultiLangDaemonConfiguration configuration = baseConfiguration();
|
||||
configuration.setCoordinatorStateWriteCapacity(defaultCoordinatorStateTableConfig.writeCapacity() + 12345);
|
||||
|
||||
final MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
|
||||
configuration.resolvedConfiguration(shardRecordProcessorFactory);
|
||||
|
||||
final CoordinatorConfig.CoordinatorStateTableConfig resolvedCoordinatorStateTableConfig =
|
||||
resolvedConfiguration.coordinatorConfig.coordinatorStateTableConfig();
|
||||
|
||||
assertNotEquals(defaultCoordinatorStateTableConfig, resolvedCoordinatorStateTableConfig);
|
||||
|
||||
// apart from the single updated configuration, all other config values should be equal to the default
|
||||
resolvedCoordinatorStateTableConfig.writeCapacity(defaultCoordinatorStateTableConfig.writeCapacity());
|
||||
assertEquals(defaultCoordinatorStateTableConfig, resolvedCoordinatorStateTableConfig);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSetLeaseTablePitrEnabledToTrue() {
|
||||
MultiLangDaemonConfiguration configuration = baseConfiguration();
|
||||
configuration.setLeaseTablePitrEnabled(true);
|
||||
|
||||
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
|
||||
configuration.resolvedConfiguration(shardRecordProcessorFactory);
|
||||
|
||||
assertTrue(resolvedConfiguration.leaseManagementConfig.leaseTablePitrEnabled());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSetLeaseTableDeletionProtectionEnabledToFalse() {
|
||||
MultiLangDaemonConfiguration configuration = baseConfiguration();
|
||||
configuration.setLeaseTableDeletionProtectionEnabled(false);
|
||||
|
||||
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
|
||||
configuration.resolvedConfiguration(shardRecordProcessorFactory);
|
||||
|
||||
assertFalse(resolvedConfiguration.leaseManagementConfig.leaseTableDeletionProtectionEnabled());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSetLeaseTablePitrEnabledToFalse() {
|
||||
MultiLangDaemonConfiguration configuration = baseConfiguration();
|
||||
configuration.setLeaseTablePitrEnabled(false);
|
||||
|
||||
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
|
||||
configuration.resolvedConfiguration(shardRecordProcessorFactory);
|
||||
|
||||
assertFalse(resolvedConfiguration.leaseManagementConfig.leaseTablePitrEnabled());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDefaultRetrievalConfig() {
|
||||
MultiLangDaemonConfiguration configuration = baseConfiguration();
|
||||
|
||||
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration
|
||||
.resolvedConfiguration(shardRecordProcessorFactory);
|
||||
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
|
||||
configuration.resolvedConfiguration(shardRecordProcessorFactory);
|
||||
|
||||
assertThat(resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(),
|
||||
instanceOf(FanOutConfig.class));
|
||||
assertThat(
|
||||
resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), instanceOf(FanOutConfig.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDefaultRetrievalConfigWithPollingConfigSet() {
|
||||
MultiLangDaemonConfiguration configuration = baseConfiguration();
|
||||
configuration.setMaxRecords(10);
|
||||
configuration.setIdleTimeBetweenReadsInMillis(60000);
|
||||
|
||||
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration
|
||||
.resolvedConfiguration(shardRecordProcessorFactory);
|
||||
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
|
||||
configuration.resolvedConfiguration(shardRecordProcessorFactory);
|
||||
|
||||
assertThat(resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(),
|
||||
instanceOf(PollingConfig.class));
|
||||
assertThat(
|
||||
resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), instanceOf(PollingConfig.class));
|
||||
assertEquals(
|
||||
10,
|
||||
((PollingConfig) resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig()).maxRecords());
|
||||
assertEquals(
|
||||
60000,
|
||||
((PollingConfig) resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig())
|
||||
.idleTimeBetweenReadsInMillis());
|
||||
assertTrue(((PollingConfig) resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig())
|
||||
.usePollingConfigIdleTimeValue());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -116,11 +387,11 @@ public class MultiLangDaemonConfigurationTest {
|
|||
MultiLangDaemonConfiguration configuration = baseConfiguration();
|
||||
configuration.setRetrievalMode(RetrievalMode.FANOUT);
|
||||
|
||||
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration
|
||||
.resolvedConfiguration(shardRecordProcessorFactory);
|
||||
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
|
||||
configuration.resolvedConfiguration(shardRecordProcessorFactory);
|
||||
|
||||
assertThat(resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(),
|
||||
instanceOf(FanOutConfig.class));
|
||||
assertThat(
|
||||
resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), instanceOf(FanOutConfig.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -128,37 +399,39 @@ public class MultiLangDaemonConfigurationTest {
|
|||
MultiLangDaemonConfiguration configuration = baseConfiguration();
|
||||
configuration.setRetrievalMode(RetrievalMode.POLLING);
|
||||
|
||||
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration
|
||||
.resolvedConfiguration(shardRecordProcessorFactory);
|
||||
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
|
||||
configuration.resolvedConfiguration(shardRecordProcessorFactory);
|
||||
|
||||
assertThat(resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(),
|
||||
instanceOf(PollingConfig.class));
|
||||
assertThat(
|
||||
resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), instanceOf(PollingConfig.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRetrievalModeSetForPollingString() throws Exception {
|
||||
MultiLangDaemonConfiguration configuration = baseConfiguration();
|
||||
|
||||
utilsBean.setProperty(configuration, "retrievalMode", RetrievalMode.POLLING.name().toLowerCase());
|
||||
utilsBean.setProperty(
|
||||
configuration, "retrievalMode", RetrievalMode.POLLING.name().toLowerCase());
|
||||
|
||||
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration
|
||||
.resolvedConfiguration(shardRecordProcessorFactory);
|
||||
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
|
||||
configuration.resolvedConfiguration(shardRecordProcessorFactory);
|
||||
|
||||
assertThat(resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(),
|
||||
instanceOf(PollingConfig.class));
|
||||
assertThat(
|
||||
resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), instanceOf(PollingConfig.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRetrievalModeSetForFanoutString() throws Exception {
|
||||
MultiLangDaemonConfiguration configuration = baseConfiguration();
|
||||
|
||||
utilsBean.setProperty(configuration, "retrievalMode", RetrievalMode.FANOUT.name().toLowerCase());
|
||||
utilsBean.setProperty(
|
||||
configuration, "retrievalMode", RetrievalMode.FANOUT.name().toLowerCase());
|
||||
|
||||
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration
|
||||
.resolvedConfiguration(shardRecordProcessorFactory);
|
||||
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
|
||||
configuration.resolvedConfiguration(shardRecordProcessorFactory);
|
||||
|
||||
assertThat(resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(),
|
||||
instanceOf(FanOutConfig.class));
|
||||
assertThat(
|
||||
resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), instanceOf(FanOutConfig.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -188,14 +461,53 @@ public class MultiLangDaemonConfigurationTest {
|
|||
configuration.setRetrievalMode(RetrievalMode.FANOUT);
|
||||
configuration.getFanoutConfig().setConsumerArn(consumerArn);
|
||||
|
||||
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration
|
||||
.resolvedConfiguration(shardRecordProcessorFactory);
|
||||
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
|
||||
configuration.resolvedConfiguration(shardRecordProcessorFactory);
|
||||
|
||||
assertThat(resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(),
|
||||
instanceOf(FanOutConfig.class));
|
||||
FanOutConfig fanOutConfig = (FanOutConfig) resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig();
|
||||
assertThat(
|
||||
resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), instanceOf(FanOutConfig.class));
|
||||
FanOutConfig fanOutConfig =
|
||||
(FanOutConfig) resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig();
|
||||
|
||||
assertThat(fanOutConfig.consumerArn(), equalTo(consumerArn));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testClientVersionConfig() {
|
||||
final CoordinatorConfig.ClientVersionConfig testClientVersionConfig =
|
||||
CoordinatorConfig.ClientVersionConfig.CLIENT_VERSION_CONFIG_COMPATIBLE_WITH_2X;
|
||||
|
||||
final MultiLangDaemonConfiguration configuration = baseConfiguration();
|
||||
configuration.setClientVersionConfig(testClientVersionConfig);
|
||||
|
||||
final MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
|
||||
configuration.resolvedConfiguration(shardRecordProcessorFactory);
|
||||
|
||||
final CoordinatorConfig coordinatorConfig = resolvedConfiguration.coordinatorConfig;
|
||||
|
||||
assertEquals(testClientVersionConfig, coordinatorConfig.clientVersionConfig());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testClientVersionConfigUsesDefault() {
|
||||
final MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
|
||||
baseConfiguration().resolvedConfiguration(shardRecordProcessorFactory);
|
||||
|
||||
final CoordinatorConfig coordinatorConfig = resolvedConfiguration.coordinatorConfig;
|
||||
|
||||
assertEquals(
|
||||
getTestConfigsBuilder().coordinatorConfig().clientVersionConfig(),
|
||||
coordinatorConfig.clientVersionConfig());
|
||||
}
|
||||
|
||||
private ConfigsBuilder getTestConfigsBuilder() {
|
||||
return new ConfigsBuilder(
|
||||
DUMMY_STREAM_NAME,
|
||||
DUMMY_APPLICATION_NAME,
|
||||
Mockito.mock(KinesisAsyncClient.class),
|
||||
Mockito.mock(DynamoDbAsyncClient.class),
|
||||
Mockito.mock(CloudWatchAsyncClient.class),
|
||||
"dummyWorkerIdentifier",
|
||||
shardRecordProcessorFactory);
|
||||
}
|
||||
}
|
||||
|
|
@ -15,6 +15,8 @@
|
|||
|
||||
package software.amazon.kinesis.multilang.config;
|
||||
|
||||
import java.util.Optional;
|
||||
|
||||
import org.apache.commons.beanutils.BeanUtilsBean;
|
||||
import org.apache.commons.beanutils.ConvertUtilsBean;
|
||||
import org.junit.Test;
|
||||
|
|
@ -24,8 +26,6 @@ import org.mockito.runners.MockitoJUnitRunner;
|
|||
import software.amazon.awssdk.services.kinesis.KinesisAsyncClient;
|
||||
import software.amazon.kinesis.retrieval.polling.PollingConfig;
|
||||
|
||||
import java.util.Optional;
|
||||
|
||||
import static org.hamcrest.CoreMatchers.equalTo;
|
||||
import static org.junit.Assert.assertThat;
|
||||
|
||||
|
|
@ -46,17 +46,23 @@ public class PollingConfigBeanTest {
|
|||
ConvertUtilsBean convertUtilsBean = new ConvertUtilsBean();
|
||||
BeanUtilsBean utilsBean = new BeanUtilsBean(convertUtilsBean);
|
||||
|
||||
MultiLangDaemonConfiguration multiLangDaemonConfiguration = new MultiLangDaemonConfiguration(utilsBean, convertUtilsBean);
|
||||
MultiLangDaemonConfiguration multiLangDaemonConfiguration =
|
||||
new MultiLangDaemonConfiguration(utilsBean, convertUtilsBean);
|
||||
multiLangDaemonConfiguration.setStreamName("test-stream");
|
||||
|
||||
PollingConfig pollingConfig = pollingConfigBean.build(kinesisAsyncClient, multiLangDaemonConfiguration);
|
||||
|
||||
assertThat(pollingConfig.kinesisClient(), equalTo(kinesisAsyncClient));
|
||||
assertThat(pollingConfig.streamName(), equalTo(multiLangDaemonConfiguration.getStreamName()));
|
||||
assertThat(pollingConfig.idleTimeBetweenReadsInMillis(), equalTo(pollingConfigBean.getIdleTimeBetweenReadsInMillis()));
|
||||
assertThat(pollingConfig.maxGetRecordsThreadPool(), equalTo(Optional.of(pollingConfigBean.getMaxGetRecordsThreadPool())));
|
||||
assertThat(
|
||||
pollingConfig.idleTimeBetweenReadsInMillis(),
|
||||
equalTo(pollingConfigBean.getIdleTimeBetweenReadsInMillis()));
|
||||
assertThat(
|
||||
pollingConfig.maxGetRecordsThreadPool(),
|
||||
equalTo(Optional.of(pollingConfigBean.getMaxGetRecordsThreadPool())));
|
||||
assertThat(pollingConfig.maxRecords(), equalTo(pollingConfigBean.getMaxRecords()));
|
||||
assertThat(pollingConfig.retryGetRecordsInSeconds(), equalTo(Optional.of(pollingConfigBean.getRetryGetRecordsInSeconds())));
|
||||
assertThat(
|
||||
pollingConfig.retryGetRecordsInSeconds(),
|
||||
equalTo(Optional.of(pollingConfigBean.getRetryGetRecordsInSeconds())));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,303 @@
|
|||
package software.amazon.kinesis.multilang.config;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.time.Duration;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import software.amazon.awssdk.services.dynamodb.model.BillingMode;
|
||||
import software.amazon.awssdk.services.dynamodb.model.Tag;
|
||||
import software.amazon.kinesis.coordinator.CoordinatorConfig.ClientVersionConfig;
|
||||
import software.amazon.kinesis.multilang.MultiLangDaemonConfig;
|
||||
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration.ResolvedConfiguration;
|
||||
import software.amazon.kinesis.processor.ShardRecordProcessor;
|
||||
import software.amazon.kinesis.processor.ShardRecordProcessorFactory;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
public class PropertiesMappingE2ETest {
|
||||
private static final String PROPERTIES_FILE = "multilang.properties";
|
||||
private static final String PROPERTIES_FILE_V3 = "multilangv3.properties";
|
||||
|
||||
@Test
|
||||
public void testKclV3PropertiesMapping() throws IOException {
|
||||
final MultiLangDaemonConfig config = new MultiLangDaemonConfig(PROPERTIES_FILE);
|
||||
|
||||
final ResolvedConfiguration kclV3Config =
|
||||
config.getMultiLangDaemonConfiguration().resolvedConfiguration(new TestRecordProcessorFactory());
|
||||
|
||||
assertEquals(
|
||||
ClientVersionConfig.CLIENT_VERSION_CONFIG_COMPATIBLE_WITH_2X,
|
||||
kclV3Config.coordinatorConfig.clientVersionConfig());
|
||||
|
||||
assertEquals(
|
||||
"MultiLangTest-CoordinatorState-CustomName",
|
||||
kclV3Config.coordinatorConfig.coordinatorStateTableConfig().tableName());
|
||||
assertEquals(
|
||||
BillingMode.PROVISIONED,
|
||||
kclV3Config.coordinatorConfig.coordinatorStateTableConfig().billingMode());
|
||||
assertEquals(
|
||||
1000,
|
||||
kclV3Config.coordinatorConfig.coordinatorStateTableConfig().readCapacity());
|
||||
assertEquals(
|
||||
500, kclV3Config.coordinatorConfig.coordinatorStateTableConfig().writeCapacity());
|
||||
assertTrue(kclV3Config.coordinatorConfig.coordinatorStateTableConfig().pointInTimeRecoveryEnabled());
|
||||
assertTrue(kclV3Config.coordinatorConfig.coordinatorStateTableConfig().deletionProtectionEnabled());
|
||||
assertEquals(
|
||||
Arrays.asList(
|
||||
Tag.builder().key("csTagK1").value("csTagV1").build(),
|
||||
Tag.builder().key("csTagK2").value("csTagV2").build(),
|
||||
Tag.builder().key("csTagK3").value("csTagV3").build()),
|
||||
kclV3Config.coordinatorConfig.coordinatorStateTableConfig().tags());
|
||||
|
||||
assertEquals(
|
||||
10000L,
|
||||
kclV3Config.leaseManagementConfig.gracefulLeaseHandoffConfig().gracefulLeaseHandoffTimeoutMillis());
|
||||
assertFalse(
|
||||
kclV3Config.leaseManagementConfig.gracefulLeaseHandoffConfig().isGracefulLeaseHandoffEnabled());
|
||||
|
||||
assertEquals(
|
||||
5000L,
|
||||
kclV3Config
|
||||
.leaseManagementConfig
|
||||
.workerUtilizationAwareAssignmentConfig()
|
||||
.inMemoryWorkerMetricsCaptureFrequencyMillis());
|
||||
assertEquals(
|
||||
60000L,
|
||||
kclV3Config
|
||||
.leaseManagementConfig
|
||||
.workerUtilizationAwareAssignmentConfig()
|
||||
.workerMetricsReporterFreqInMillis());
|
||||
assertEquals(
|
||||
50,
|
||||
kclV3Config
|
||||
.leaseManagementConfig
|
||||
.workerUtilizationAwareAssignmentConfig()
|
||||
.noOfPersistedMetricsPerWorkerMetrics());
|
||||
assertTrue(kclV3Config
|
||||
.leaseManagementConfig
|
||||
.workerUtilizationAwareAssignmentConfig()
|
||||
.disableWorkerMetrics());
|
||||
assertEquals(
|
||||
10000,
|
||||
kclV3Config
|
||||
.leaseManagementConfig
|
||||
.workerUtilizationAwareAssignmentConfig()
|
||||
.maxThroughputPerHostKBps());
|
||||
assertEquals(
|
||||
90,
|
||||
kclV3Config
|
||||
.leaseManagementConfig
|
||||
.workerUtilizationAwareAssignmentConfig()
|
||||
.dampeningPercentage());
|
||||
assertEquals(
|
||||
5,
|
||||
kclV3Config
|
||||
.leaseManagementConfig
|
||||
.workerUtilizationAwareAssignmentConfig()
|
||||
.reBalanceThresholdPercentage());
|
||||
assertFalse(kclV3Config
|
||||
.leaseManagementConfig
|
||||
.workerUtilizationAwareAssignmentConfig()
|
||||
.allowThroughputOvershoot());
|
||||
assertEquals(
|
||||
Duration.ofHours(12),
|
||||
kclV3Config
|
||||
.leaseManagementConfig
|
||||
.workerUtilizationAwareAssignmentConfig()
|
||||
.staleWorkerMetricsEntryCleanupDuration());
|
||||
assertEquals(
|
||||
5,
|
||||
kclV3Config
|
||||
.leaseManagementConfig
|
||||
.workerUtilizationAwareAssignmentConfig()
|
||||
.varianceBalancingFrequency());
|
||||
assertEquals(
|
||||
0.18D,
|
||||
kclV3Config
|
||||
.leaseManagementConfig
|
||||
.workerUtilizationAwareAssignmentConfig()
|
||||
.workerMetricsEMAAlpha());
|
||||
|
||||
assertEquals(
|
||||
"MultiLangTest-WorkerMetrics-CustomName",
|
||||
kclV3Config
|
||||
.leaseManagementConfig
|
||||
.workerUtilizationAwareAssignmentConfig()
|
||||
.workerMetricsTableConfig()
|
||||
.tableName());
|
||||
assertEquals(
|
||||
BillingMode.PROVISIONED,
|
||||
kclV3Config
|
||||
.leaseManagementConfig
|
||||
.workerUtilizationAwareAssignmentConfig()
|
||||
.workerMetricsTableConfig()
|
||||
.billingMode());
|
||||
assertEquals(
|
||||
250,
|
||||
kclV3Config
|
||||
.leaseManagementConfig
|
||||
.workerUtilizationAwareAssignmentConfig()
|
||||
.workerMetricsTableConfig()
|
||||
.readCapacity());
|
||||
assertEquals(
|
||||
90,
|
||||
kclV3Config
|
||||
.leaseManagementConfig
|
||||
.workerUtilizationAwareAssignmentConfig()
|
||||
.workerMetricsTableConfig()
|
||||
.writeCapacity());
|
||||
assertTrue(kclV3Config
|
||||
.leaseManagementConfig
|
||||
.workerUtilizationAwareAssignmentConfig()
|
||||
.workerMetricsTableConfig()
|
||||
.pointInTimeRecoveryEnabled());
|
||||
assertTrue(kclV3Config
|
||||
.leaseManagementConfig
|
||||
.workerUtilizationAwareAssignmentConfig()
|
||||
.workerMetricsTableConfig()
|
||||
.deletionProtectionEnabled());
|
||||
assertEquals(
|
||||
Arrays.asList(
|
||||
Tag.builder().key("wmTagK1").value("wmTagV1").build(),
|
||||
Tag.builder().key("wmTagK2").value("wmTagV2").build()),
|
||||
kclV3Config
|
||||
.leaseManagementConfig
|
||||
.workerUtilizationAwareAssignmentConfig()
|
||||
.workerMetricsTableConfig()
|
||||
.tags());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testKclV3PropertiesMappingForDefaultValues() throws IOException {
|
||||
final MultiLangDaemonConfig config = new MultiLangDaemonConfig(PROPERTIES_FILE_V3);
|
||||
|
||||
final ResolvedConfiguration kclV3Config =
|
||||
config.getMultiLangDaemonConfiguration().resolvedConfiguration(new TestRecordProcessorFactory());
|
||||
|
||||
assertEquals(ClientVersionConfig.CLIENT_VERSION_CONFIG_3X, kclV3Config.coordinatorConfig.clientVersionConfig());
|
||||
|
||||
assertEquals(
|
||||
"MultiLangTest-CoordinatorState",
|
||||
kclV3Config.coordinatorConfig.coordinatorStateTableConfig().tableName());
|
||||
assertEquals(
|
||||
BillingMode.PAY_PER_REQUEST,
|
||||
kclV3Config.coordinatorConfig.coordinatorStateTableConfig().billingMode());
|
||||
assertFalse(kclV3Config.coordinatorConfig.coordinatorStateTableConfig().pointInTimeRecoveryEnabled());
|
||||
assertFalse(kclV3Config.coordinatorConfig.coordinatorStateTableConfig().deletionProtectionEnabled());
|
||||
assertEquals(
|
||||
Collections.emptyList(),
|
||||
kclV3Config.coordinatorConfig.coordinatorStateTableConfig().tags());
|
||||
|
||||
assertEquals(
|
||||
30_000L,
|
||||
kclV3Config.leaseManagementConfig.gracefulLeaseHandoffConfig().gracefulLeaseHandoffTimeoutMillis());
|
||||
assertTrue(
|
||||
kclV3Config.leaseManagementConfig.gracefulLeaseHandoffConfig().isGracefulLeaseHandoffEnabled());
|
||||
|
||||
assertEquals(
|
||||
1000L,
|
||||
kclV3Config
|
||||
.leaseManagementConfig
|
||||
.workerUtilizationAwareAssignmentConfig()
|
||||
.inMemoryWorkerMetricsCaptureFrequencyMillis());
|
||||
assertEquals(
|
||||
30000L,
|
||||
kclV3Config
|
||||
.leaseManagementConfig
|
||||
.workerUtilizationAwareAssignmentConfig()
|
||||
.workerMetricsReporterFreqInMillis());
|
||||
assertEquals(
|
||||
10,
|
||||
kclV3Config
|
||||
.leaseManagementConfig
|
||||
.workerUtilizationAwareAssignmentConfig()
|
||||
.noOfPersistedMetricsPerWorkerMetrics());
|
||||
assertFalse(kclV3Config
|
||||
.leaseManagementConfig
|
||||
.workerUtilizationAwareAssignmentConfig()
|
||||
.disableWorkerMetrics());
|
||||
assertEquals(
|
||||
Double.MAX_VALUE,
|
||||
kclV3Config
|
||||
.leaseManagementConfig
|
||||
.workerUtilizationAwareAssignmentConfig()
|
||||
.maxThroughputPerHostKBps());
|
||||
assertEquals(
|
||||
60,
|
||||
kclV3Config
|
||||
.leaseManagementConfig
|
||||
.workerUtilizationAwareAssignmentConfig()
|
||||
.dampeningPercentage());
|
||||
assertEquals(
|
||||
10,
|
||||
kclV3Config
|
||||
.leaseManagementConfig
|
||||
.workerUtilizationAwareAssignmentConfig()
|
||||
.reBalanceThresholdPercentage());
|
||||
assertTrue(kclV3Config
|
||||
.leaseManagementConfig
|
||||
.workerUtilizationAwareAssignmentConfig()
|
||||
.allowThroughputOvershoot());
|
||||
assertEquals(
|
||||
Duration.ofDays(1),
|
||||
kclV3Config
|
||||
.leaseManagementConfig
|
||||
.workerUtilizationAwareAssignmentConfig()
|
||||
.staleWorkerMetricsEntryCleanupDuration());
|
||||
assertEquals(
|
||||
3,
|
||||
kclV3Config
|
||||
.leaseManagementConfig
|
||||
.workerUtilizationAwareAssignmentConfig()
|
||||
.varianceBalancingFrequency());
|
||||
assertEquals(
|
||||
0.5D,
|
||||
kclV3Config
|
||||
.leaseManagementConfig
|
||||
.workerUtilizationAwareAssignmentConfig()
|
||||
.workerMetricsEMAAlpha());
|
||||
|
||||
assertEquals(
|
||||
"MultiLangTest-WorkerMetricStats",
|
||||
kclV3Config
|
||||
.leaseManagementConfig
|
||||
.workerUtilizationAwareAssignmentConfig()
|
||||
.workerMetricsTableConfig()
|
||||
.tableName());
|
||||
assertEquals(
|
||||
BillingMode.PAY_PER_REQUEST,
|
||||
kclV3Config
|
||||
.leaseManagementConfig
|
||||
.workerUtilizationAwareAssignmentConfig()
|
||||
.workerMetricsTableConfig()
|
||||
.billingMode());
|
||||
assertFalse(kclV3Config
|
||||
.leaseManagementConfig
|
||||
.workerUtilizationAwareAssignmentConfig()
|
||||
.workerMetricsTableConfig()
|
||||
.pointInTimeRecoveryEnabled());
|
||||
assertFalse(kclV3Config
|
||||
.leaseManagementConfig
|
||||
.workerUtilizationAwareAssignmentConfig()
|
||||
.workerMetricsTableConfig()
|
||||
.deletionProtectionEnabled());
|
||||
assertEquals(
|
||||
Collections.emptyList(),
|
||||
kclV3Config
|
||||
.leaseManagementConfig
|
||||
.workerUtilizationAwareAssignmentConfig()
|
||||
.workerMetricsTableConfig()
|
||||
.tags());
|
||||
}
|
||||
|
||||
private static class TestRecordProcessorFactory implements ShardRecordProcessorFactory {
|
||||
@Override
|
||||
public ShardRecordProcessor shardRecordProcessor() {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
/*
|
||||
* Copyright 2024 Amazon.com, Inc. or its affiliates.
|
||||
* Licensed under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package software.amazon.kinesis.multilang.config;
|
||||
|
||||
import java.util.Optional;
|
||||
|
||||
import org.apache.commons.beanutils.BeanUtilsBean;
|
||||
import org.apache.commons.beanutils.ConvertUtilsBean;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.runners.MockitoJUnitRunner;
|
||||
import software.amazon.awssdk.services.kinesis.KinesisAsyncClient;
|
||||
import software.amazon.kinesis.retrieval.polling.PollingConfig;
|
||||
|
||||
import static org.hamcrest.CoreMatchers.equalTo;
|
||||
import static org.junit.Assert.assertThat;
|
||||
|
||||
@RunWith(MockitoJUnitRunner.class)
|
||||
public class WorkerUtilizationAwareAssignmentConfigBeanTest {
|
||||
|
||||
@Mock
|
||||
private KinesisAsyncClient kinesisAsyncClient;
|
||||
|
||||
@Test
|
||||
public void testAllPropertiesTransit() {
|
||||
PollingConfigBean pollingConfigBean = new PollingConfigBean();
|
||||
pollingConfigBean.setIdleTimeBetweenReadsInMillis(1000);
|
||||
pollingConfigBean.setMaxGetRecordsThreadPool(20);
|
||||
pollingConfigBean.setMaxRecords(5000);
|
||||
pollingConfigBean.setRetryGetRecordsInSeconds(30);
|
||||
|
||||
ConvertUtilsBean convertUtilsBean = new ConvertUtilsBean();
|
||||
BeanUtilsBean utilsBean = new BeanUtilsBean(convertUtilsBean);
|
||||
|
||||
MultiLangDaemonConfiguration multiLangDaemonConfiguration =
|
||||
new MultiLangDaemonConfiguration(utilsBean, convertUtilsBean);
|
||||
multiLangDaemonConfiguration.setStreamName("test-stream");
|
||||
|
||||
PollingConfig pollingConfig = pollingConfigBean.build(kinesisAsyncClient, multiLangDaemonConfiguration);
|
||||
|
||||
assertThat(pollingConfig.kinesisClient(), equalTo(kinesisAsyncClient));
|
||||
assertThat(pollingConfig.streamName(), equalTo(multiLangDaemonConfiguration.getStreamName()));
|
||||
assertThat(
|
||||
pollingConfig.idleTimeBetweenReadsInMillis(),
|
||||
equalTo(pollingConfigBean.getIdleTimeBetweenReadsInMillis()));
|
||||
assertThat(
|
||||
pollingConfig.maxGetRecordsThreadPool(),
|
||||
equalTo(Optional.of(pollingConfigBean.getMaxGetRecordsThreadPool())));
|
||||
assertThat(pollingConfig.maxRecords(), equalTo(pollingConfigBean.getMaxRecords()));
|
||||
assertThat(
|
||||
pollingConfig.retryGetRecordsInSeconds(),
|
||||
equalTo(Optional.of(pollingConfigBean.getRetryGetRecordsInSeconds())));
|
||||
}
|
||||
}
|
||||
|
|
@ -15,11 +15,6 @@
|
|||
|
||||
package software.amazon.kinesis.multilang.messages;
|
||||
|
||||
import static org.hamcrest.CoreMatchers.equalTo;
|
||||
import static org.hamcrest.CoreMatchers.nullValue;
|
||||
import static org.hamcrest.CoreMatchers.sameInstance;
|
||||
import static org.junit.Assert.assertThat;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.time.Instant;
|
||||
import java.util.Arrays;
|
||||
|
|
@ -31,9 +26,13 @@ import org.hamcrest.Description;
|
|||
import org.hamcrest.Matcher;
|
||||
import org.hamcrest.TypeSafeDiagnosingMatcher;
|
||||
import org.junit.Test;
|
||||
|
||||
import software.amazon.kinesis.retrieval.KinesisClientRecord;
|
||||
|
||||
import static org.hamcrest.CoreMatchers.equalTo;
|
||||
import static org.hamcrest.CoreMatchers.nullValue;
|
||||
import static org.hamcrest.CoreMatchers.sameInstance;
|
||||
import static org.junit.Assert.assertThat;
|
||||
|
||||
public class JsonFriendlyRecordTest {
|
||||
|
||||
private KinesisClientRecord kinesisClientRecord;
|
||||
|
|
@ -82,14 +81,15 @@ public class JsonFriendlyRecordTest {
|
|||
|
||||
private RecordMatcher(KinesisClientRecord expected) {
|
||||
this.matchers = Arrays.asList(
|
||||
new FieldMatcher<>("approximateArrivalTimestamp",
|
||||
new FieldMatcher<>(
|
||||
"approximateArrivalTimestamp",
|
||||
equalTo(expected.approximateArrivalTimestamp().toEpochMilli()),
|
||||
JsonFriendlyRecord::getApproximateArrivalTimestamp),
|
||||
new FieldMatcher<>("partitionKey", expected::partitionKey, JsonFriendlyRecord::getPartitionKey),
|
||||
new FieldMatcher<>("sequenceNumber", expected::sequenceNumber,
|
||||
JsonFriendlyRecord::getSequenceNumber),
|
||||
new FieldMatcher<>("subSequenceNumber", expected::subSequenceNumber,
|
||||
JsonFriendlyRecord::getSubSequenceNumber),
|
||||
new FieldMatcher<>(
|
||||
"sequenceNumber", expected::sequenceNumber, JsonFriendlyRecord::getSequenceNumber),
|
||||
new FieldMatcher<>(
|
||||
"subSequenceNumber", expected::subSequenceNumber, JsonFriendlyRecord::getSubSequenceNumber),
|
||||
new FieldMatcher<>("data", dataEquivalentTo(expected.data()), JsonFriendlyRecord::getData));
|
||||
|
||||
this.expected = expected;
|
||||
|
|
@ -97,13 +97,16 @@ public class JsonFriendlyRecordTest {
|
|||
|
||||
@Override
|
||||
protected boolean matchesSafely(JsonFriendlyRecord item, Description mismatchDescription) {
|
||||
return matchers.stream().map(m -> {
|
||||
return matchers.stream()
|
||||
.map(m -> {
|
||||
if (!m.matches(item)) {
|
||||
m.describeMismatch(item, mismatchDescription);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}).reduce((l, r) -> l && r).orElse(true);
|
||||
})
|
||||
.reduce((l, r) -> l && r)
|
||||
.orElse(true);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -160,8 +163,9 @@ public class JsonFriendlyRecordTest {
|
|||
}
|
||||
|
||||
private KinesisClientRecord.KinesisClientRecordBuilder defaultRecord() {
|
||||
return KinesisClientRecord.builder().partitionKey("test-partition").sequenceNumber("123")
|
||||
return KinesisClientRecord.builder()
|
||||
.partitionKey("test-partition")
|
||||
.sequenceNumber("123")
|
||||
.approximateArrivalTimestamp(Instant.now());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -17,15 +17,13 @@ package software.amazon.kinesis.multilang.messages;
|
|||
import java.nio.ByteBuffer;
|
||||
import java.util.Collections;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import software.amazon.kinesis.lifecycle.ShutdownReason;
|
||||
import software.amazon.kinesis.lifecycle.events.InitializationInput;
|
||||
import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput;
|
||||
import software.amazon.kinesis.lifecycle.ShutdownReason;
|
||||
import software.amazon.kinesis.retrieval.KinesisClientRecord;
|
||||
|
||||
public class MessageTest {
|
||||
|
|
@ -34,10 +32,10 @@ public class MessageTest {
|
|||
public void toStringTest() {
|
||||
Message[] messages = new Message[] {
|
||||
new CheckpointMessage("1234567890", 0L, null),
|
||||
new InitializeMessage(InitializationInput.builder().shardId("shard-123").build()),
|
||||
new InitializeMessage(
|
||||
InitializationInput.builder().shardId("shard-123").build()),
|
||||
new ProcessRecordsMessage(ProcessRecordsInput.builder()
|
||||
.records(Collections.singletonList(
|
||||
KinesisClientRecord.builder()
|
||||
.records(Collections.singletonList(KinesisClientRecord.builder()
|
||||
.data(ByteBuffer.wrap("cat".getBytes()))
|
||||
.partitionKey("cat")
|
||||
.sequenceNumber("555")
|
||||
|
|
@ -55,19 +53,21 @@ public class MessageTest {
|
|||
// TODO: fix this
|
||||
for (int i = 0; i < messages.length; i++) {
|
||||
System.out.println(messages[i].toString());
|
||||
Assert.assertTrue("Each message should contain the action field", messages[i].toString().contains("action"));
|
||||
Assert.assertTrue(
|
||||
"Each message should contain the action field",
|
||||
messages[i].toString().contains("action"));
|
||||
}
|
||||
|
||||
// Hit this constructor
|
||||
KinesisClientRecord defaultJsonFriendlyRecord = KinesisClientRecord.builder().build();
|
||||
KinesisClientRecord defaultJsonFriendlyRecord =
|
||||
KinesisClientRecord.builder().build();
|
||||
Assert.assertNull(defaultJsonFriendlyRecord.partitionKey());
|
||||
Assert.assertNull(defaultJsonFriendlyRecord.data());
|
||||
Assert.assertNull(defaultJsonFriendlyRecord.sequenceNumber());
|
||||
Assert.assertNull(new ShutdownMessage(null).getReason());
|
||||
|
||||
// Hit the bad object mapping path
|
||||
Message withBadMapper = new Message() {
|
||||
}.withObjectMapper(new ObjectMapper() {
|
||||
Message withBadMapper = new Message() {}.withObjectMapper(new ObjectMapper() {
|
||||
/**
|
||||
*
|
||||
*/
|
||||
|
|
@ -75,8 +75,7 @@ public class MessageTest {
|
|||
|
||||
@Override
|
||||
public String writeValueAsString(Object m) throws JsonProcessingException {
|
||||
throw new JsonProcessingException(new Throwable()) {
|
||||
};
|
||||
throw new JsonProcessingException(new Throwable()) {};
|
||||
}
|
||||
});
|
||||
String s = withBadMapper.toString();
|
||||
|
|
|
|||
|
|
@ -0,0 +1,175 @@
|
|||
# The script that abides by the multi-language protocol. This script will
|
||||
# be executed by the MultiLangDaemon, which will communicate with this script
|
||||
# over STDIN and STDOUT according to the multi-language protocol.
|
||||
executableName = sample_kclpy_app.py
|
||||
|
||||
# The Stream arn: arn:aws:kinesis:<region>:<account id>:stream/<stream name>
|
||||
# Important: streamArn takes precedence over streamName if both are set
|
||||
streamArn = arn:aws:kinesis:us-east-5:000000000000:stream/kclpysample
|
||||
|
||||
# The name of an Amazon Kinesis stream to process.
|
||||
# Important: streamArn takes precedence over streamName if both are set
|
||||
streamName = kclpysample
|
||||
|
||||
# Used by the KCL as the name of this application. Will be used as the name
|
||||
# of an Amazon DynamoDB table which will store the lease and checkpoint
|
||||
# information for workers with this application name
|
||||
applicationName = MultiLangTest
|
||||
|
||||
# Users can change the credentials provider the KCL will use to retrieve credentials.
|
||||
# Expected key name (case-sensitive):
|
||||
# AwsCredentialsProvider / AwsCredentialsProviderDynamoDB / AwsCredentialsProviderCloudWatch
|
||||
# The DefaultCredentialsProvider checks several other providers, which is
|
||||
# described here:
|
||||
# https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/auth/credentials/DefaultCredentialsProvider.html
|
||||
AwsCredentialsProvider = DefaultCredentialsProvider
|
||||
|
||||
# Appended to the user agent of the KCL. Does not impact the functionality of the
|
||||
# KCL in any other way.
|
||||
processingLanguage = python/3.8
|
||||
|
||||
# Valid options at TRIM_HORIZON or LATEST.
|
||||
# See http://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetShardIterator.html#API_GetShardIterator_RequestSyntax
|
||||
initialPositionInStream = TRIM_HORIZON
|
||||
|
||||
# To specify an initial timestamp from which to start processing records, please specify timestamp value for 'initiatPositionInStreamExtended',
|
||||
# and uncomment below line with right timestamp value.
|
||||
# See more from 'Timestamp' under http://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetShardIterator.html#API_GetShardIterator_RequestSyntax
|
||||
#initialPositionInStreamExtended = 1636609142
|
||||
|
||||
# The following properties are also available for configuring the KCL Worker that is created
|
||||
# by the MultiLangDaemon.
|
||||
|
||||
# The KCL defaults to us-east-1
|
||||
regionName = us-east-1
|
||||
|
||||
# Fail over time in milliseconds. A worker which does not renew it's lease within this time interval
|
||||
# will be regarded as having problems and it's shards will be assigned to other workers.
|
||||
# For applications that have a large number of shards, this msy be set to a higher number to reduce
|
||||
# the number of DynamoDB IOPS required for tracking leases
|
||||
failoverTimeMillis = 10000
|
||||
|
||||
# A worker id that uniquely identifies this worker among all workers using the same applicationName
|
||||
# If this isn't provided a MultiLangDaemon instance will assign a unique workerId to itself.
|
||||
workerId = "workerId"
|
||||
|
||||
# Shard sync interval in milliseconds - e.g. wait for this long between shard sync tasks.
|
||||
shardSyncIntervalMillis = 60000
|
||||
|
||||
# Max records to fetch from Kinesis in a single GetRecords call.
|
||||
maxRecords = 10000
|
||||
|
||||
# Idle time between record reads in milliseconds.
|
||||
idleTimeBetweenReadsInMillis = 1000
|
||||
|
||||
# Enables applications flush/checkpoint (if they have some data "in progress", but don't get new data for while)
|
||||
callProcessRecordsEvenForEmptyRecordList = false
|
||||
|
||||
# Interval in milliseconds between polling to check for parent shard completion.
|
||||
# Polling frequently will take up more DynamoDB IOPS (when there are leases for shards waiting on
|
||||
# completion of parent shards).
|
||||
parentShardPollIntervalMillis = 10000
|
||||
|
||||
# Cleanup leases upon shards completion (don't wait until they expire in Kinesis).
|
||||
# Keeping leases takes some tracking/resources (e.g. they need to be renewed, assigned), so by default we try
|
||||
# to delete the ones we don't need any longer.
|
||||
cleanupLeasesUponShardCompletion = true
|
||||
|
||||
# Backoff time in milliseconds for Amazon Kinesis Client Library tasks (in the event of failures).
|
||||
taskBackoffTimeMillis = 500
|
||||
|
||||
# Buffer metrics for at most this long before publishing to CloudWatch.
|
||||
metricsBufferTimeMillis = 10000
|
||||
|
||||
# Buffer at most this many metrics before publishing to CloudWatch.
|
||||
metricsMaxQueueSize = 10000
|
||||
|
||||
# KCL will validate client provided sequence numbers with a call to Amazon Kinesis before checkpointing for calls
|
||||
# to RecordProcessorCheckpointer#checkpoint(String) by default.
|
||||
validateSequenceNumberBeforeCheckpointing = true
|
||||
|
||||
# The maximum number of active threads for the MultiLangDaemon to permit.
|
||||
# If a value is provided then a FixedThreadPool is used with the maximum
|
||||
# active threads set to the provided value. If a non-positive integer or no
|
||||
# value is provided a CachedThreadPool is used.
|
||||
maxActiveThreads = -1
|
||||
|
||||
################### KclV3 configurations ###################
|
||||
# NOTE : These are just test configurations to show how to customize
|
||||
# all possible KCLv3 configurations. They are not necessarily the best
|
||||
# default values to use for production.
|
||||
|
||||
# Coordinator config
|
||||
# Version the KCL needs to operate in. For more details check the KCLv3 migration
|
||||
# documentation. Default is CLIENT_VERSION_CONFIG_3X
|
||||
clientVersionConfig = CLIENT_VERSION_CONFIG_COMPATIBLE_WITH_2x
|
||||
# Configurations to control how the CoordinatorState DDB table is created
|
||||
# Default name is applicationName-CoordinatorState in PAY_PER_REQUEST,
|
||||
# with PITR and deletion protection disabled and no tags
|
||||
coordinatorStateTableName = MultiLangTest-CoordinatorState-CustomName
|
||||
coordinatorStateBillingMode = PROVISIONED
|
||||
coordinatorStateReadCapacity = 1000
|
||||
coordinatorStateWriteCapacity = 500
|
||||
coordinatorStatePointInTimeRecoveryEnabled = true
|
||||
coordinatorStateDeletionProtectionEnabled = true
|
||||
coordinatorStateTags = csTagK1=csTagV1,csTagK2=csTagV2,csTagK3=csTagV3
|
||||
|
||||
# Graceful handoff config - tuning of the shutdown behavior during lease transfers
|
||||
# default values are 30000 and true respectively
|
||||
gracefulLeaseHandoffTimeoutMillis = 10000
|
||||
isGracefulLeaseHandoffEnabled = false
|
||||
|
||||
# WorkerMetricStats table config - control how the DDB table is created
|
||||
# Default name is applicationName-WorkerMetricStats in PAY_PER_REQUEST,
|
||||
# with PITR and deletion protection disabled and no tags
|
||||
workerMetricsTableName = MultiLangTest-WorkerMetrics-CustomName
|
||||
workerMetricsBillingMode = PROVISIONED
|
||||
workerMetricsReadCapacity = 250
|
||||
workerMetricsWriteCapacity = 90
|
||||
workerMetricsPointInTimeRecoveryEnabled = true
|
||||
workerMetricsDeletionProtectionEnabled = true
|
||||
workerMetricsTags = wmTagK1=wmTagV1,wmTagK2=wmTagV2
|
||||
|
||||
# WorkerUtilizationAwareAssignment config - tune the new KCLv3 Lease balancing algorithm
|
||||
#
|
||||
# frequency of capturing worker metrics in memory. Default is 1s
|
||||
inMemoryWorkerMetricsCaptureFrequencyMillis = 5000
|
||||
# frequency of reporting worker metric stats to storage. Default is 30s
|
||||
workerMetricsReporterFreqInMillis = 60000
|
||||
# No. of metricStats that are persisted in WorkerMetricStats ddb table, default is 10
|
||||
noOfPersistedMetricsPerWorkerMetrics = 50
|
||||
# Disable use of worker metrics to balance lease, default is false.
|
||||
# If it is true, the algorithm balances lease based on worker's processing throughput.
|
||||
disableWorkerMetrics = true
|
||||
# Max throughput per host 10 MBps, to limit processing to the given value
|
||||
# Default is unlimited.
|
||||
maxThroughputPerHostKBps = 10000
|
||||
# Dampen the load that is rebalanced during lease re-balancing, default is 60%
|
||||
dampeningPercentage = 90
|
||||
# Configures the allowed variance range for worker utilization. The upper
|
||||
# limit is calculated as average * (1 + reBalanceThresholdPercentage/100).
|
||||
# The lower limit is average * (1 - reBalanceThresholdPercentage/100). If
|
||||
# any worker's utilization falls outside this range, lease re-balancing is
|
||||
# triggered. The re-balancing algorithm aims to bring variance within the
|
||||
# specified range. It also avoids thrashing by ensuring the utilization of
|
||||
# the worker receiving the load after re-balancing doesn't exceed the fleet
|
||||
# average. This might cause no re-balancing action even the utilization is
|
||||
# out of the variance range. The default value is 10, representing +/-10%
|
||||
# variance from the average value.
|
||||
reBalanceThresholdPercentage = 5
|
||||
# Whether at-least one lease must be taken from a high utilization worker
|
||||
# during re-balancing when there is no lease assigned to that worker which has
|
||||
# throughput is less than or equal to the minimum throughput that needs to be
|
||||
# moved away from that worker to bring the worker back into the allowed variance.
|
||||
# Default is true.
|
||||
allowThroughputOvershoot = false
|
||||
# Lease assignment is performed every failoverTimeMillis but re-balance will
|
||||
# be attempted only once in 5 times based on the below config. Default is 3.
|
||||
varianceBalancingFrequency = 5
|
||||
# Alpha value used for calculating exponential moving average of worker's metricStats.
|
||||
workerMetricsEMAAlpha = 0.18
|
||||
# Duration after which workerMetricStats entry from WorkerMetricStats table will
|
||||
# be cleaned up.
|
||||
# Duration format examples: PT15M (15 mins) PT10H (10 hours) P2D (2 days)
|
||||
# Refer to Duration.parse javadocs for more details
|
||||
staleWorkerMetricsEntryCleanupDuration = PT12H
|
||||
|
|
@ -0,0 +1,100 @@
|
|||
# The script that abides by the multi-language protocol. This script will
|
||||
# be executed by the MultiLangDaemon, which will communicate with this script
|
||||
# over STDIN and STDOUT according to the multi-language protocol.
|
||||
executableName = sample_kclpy_app.py
|
||||
|
||||
# The Stream arn: arn:aws:kinesis:<region>:<account id>:stream/<stream name>
|
||||
# Important: streamArn takes precedence over streamName if both are set
|
||||
streamArn = arn:aws:kinesis:us-east-5:000000000000:stream/kclpysample
|
||||
|
||||
# The name of an Amazon Kinesis stream to process.
|
||||
# Important: streamArn takes precedence over streamName if both are set
|
||||
streamName = kclpysample
|
||||
|
||||
# Used by the KCL as the name of this application. Will be used as the name
|
||||
# of an Amazon DynamoDB table which will store the lease and checkpoint
|
||||
# information for workers with this application name
|
||||
applicationName = MultiLangTest
|
||||
|
||||
# Users can change the credentials provider the KCL will use to retrieve credentials.
|
||||
# Expected key name (case-sensitive):
|
||||
# AwsCredentialsProvider / AwsCredentialsProviderDynamoDB / AwsCredentialsProviderCloudWatch
|
||||
# The DefaultCredentialsProvider checks several other providers, which is
|
||||
# described here:
|
||||
# https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/auth/credentials/DefaultCredentialsProvider.html
|
||||
AwsCredentialsProvider = DefaultCredentialsProvider
|
||||
|
||||
# Appended to the user agent of the KCL. Does not impact the functionality of the
|
||||
# KCL in any other way.
|
||||
processingLanguage = python/3.8
|
||||
|
||||
# Valid options at TRIM_HORIZON or LATEST.
|
||||
# See http://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetShardIterator.html#API_GetShardIterator_RequestSyntax
|
||||
initialPositionInStream = TRIM_HORIZON
|
||||
|
||||
# To specify an initial timestamp from which to start processing records, please specify timestamp value for 'initiatPositionInStreamExtended',
|
||||
# and uncomment below line with right timestamp value.
|
||||
# See more from 'Timestamp' under http://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetShardIterator.html#API_GetShardIterator_RequestSyntax
|
||||
#initialPositionInStreamExtended = 1636609142
|
||||
|
||||
# The following properties are also available for configuring the KCL Worker that is created
|
||||
# by the MultiLangDaemon.
|
||||
|
||||
# The KCL defaults to us-east-1
|
||||
regionName = us-east-1
|
||||
|
||||
# Fail over time in milliseconds. A worker which does not renew it's lease within this time interval
|
||||
# will be regarded as having problems and it's shards will be assigned to other workers.
|
||||
# For applications that have a large number of shards, this msy be set to a higher number to reduce
|
||||
# the number of DynamoDB IOPS required for tracking leases
|
||||
failoverTimeMillis = 10000
|
||||
|
||||
# A worker id that uniquely identifies this worker among all workers using the same applicationName
|
||||
# If this isn't provided a MultiLangDaemon instance will assign a unique workerId to itself.
|
||||
workerId = "workerId"
|
||||
|
||||
# Shard sync interval in milliseconds - e.g. wait for this long between shard sync tasks.
|
||||
shardSyncIntervalMillis = 60000
|
||||
|
||||
# Max records to fetch from Kinesis in a single GetRecords call.
|
||||
maxRecords = 10000
|
||||
|
||||
# Idle time between record reads in milliseconds.
|
||||
idleTimeBetweenReadsInMillis = 1000
|
||||
|
||||
# Enables applications flush/checkpoint (if they have some data "in progress", but don't get new data for while)
|
||||
callProcessRecordsEvenForEmptyRecordList = false
|
||||
|
||||
# Interval in milliseconds between polling to check for parent shard completion.
|
||||
# Polling frequently will take up more DynamoDB IOPS (when there are leases for shards waiting on
|
||||
# completion of parent shards).
|
||||
parentShardPollIntervalMillis = 10000
|
||||
|
||||
# Cleanup leases upon shards completion (don't wait until they expire in Kinesis).
|
||||
# Keeping leases takes some tracking/resources (e.g. they need to be renewed, assigned), so by default we try
|
||||
# to delete the ones we don't need any longer.
|
||||
cleanupLeasesUponShardCompletion = true
|
||||
|
||||
# Backoff time in milliseconds for Amazon Kinesis Client Library tasks (in the event of failures).
|
||||
taskBackoffTimeMillis = 500
|
||||
|
||||
# Buffer metrics for at most this long before publishing to CloudWatch.
|
||||
metricsBufferTimeMillis = 10000
|
||||
|
||||
# Buffer at most this many metrics before publishing to CloudWatch.
|
||||
metricsMaxQueueSize = 10000
|
||||
|
||||
# KCL will validate client provided sequence numbers with a call to Amazon Kinesis before checkpointing for calls
|
||||
# to RecordProcessorCheckpointer#checkpoint(String) by default.
|
||||
validateSequenceNumberBeforeCheckpointing = true
|
||||
|
||||
# The maximum number of active threads for the MultiLangDaemon to permit.
|
||||
# If a value is provided then a FixedThreadPool is used with the maximum
|
||||
# active threads set to the provided value. If a non-positive integer or no
|
||||
# value is provided a CachedThreadPool is used.
|
||||
maxActiveThreads = -1
|
||||
|
||||
################### KclV3 configurations ###################
|
||||
# Coordinator config
|
||||
clientVersionConfig = CLIENT_VERSION_CONFIG_3x
|
||||
## Let all other KCLv3 config use defaults
|
||||
|
|
@ -1,4 +1,5 @@
|
|||
<!--
|
||||
|
||||
/*
|
||||
* Copyright 2019 Amazon.com, Inc. or its affiliates.
|
||||
* Licensed under the Apache License, Version 2.0 (the
|
||||
|
|
@ -22,7 +23,7 @@
|
|||
<parent>
|
||||
<groupId>software.amazon.kinesis</groupId>
|
||||
<artifactId>amazon-kinesis-client-pom</artifactId>
|
||||
<version>2.5.2-SNAPSHOT</version>
|
||||
<version>3.0.3</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>amazon-kinesis-client</artifactId>
|
||||
|
|
@ -47,11 +48,13 @@
|
|||
</licenses>
|
||||
|
||||
<properties>
|
||||
<protobuf.version>4.27.5</protobuf.version>
|
||||
<sqlite4java.version>1.0.392</sqlite4java.version>
|
||||
<sqlite4java.native>libsqlite4java</sqlite4java.native>
|
||||
<sqlite4java.libpath>${project.build.directory}/test-lib</sqlite4java.libpath>
|
||||
<slf4j.version>2.0.7</slf4j.version>
|
||||
<gsr.version>1.1.14</gsr.version>
|
||||
<slf4j.version>2.0.13</slf4j.version>
|
||||
<gsr.version>1.1.19</gsr.version>
|
||||
<skipITs>true</skipITs>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
|
|
@ -65,6 +68,18 @@
|
|||
<artifactId>dynamodb</artifactId>
|
||||
<version>${awssdk.version}</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/software.amazon.awssdk/dynamodb-enhanced -->
|
||||
<dependency>
|
||||
<groupId>software.amazon.awssdk</groupId>
|
||||
<artifactId>dynamodb-enhanced</artifactId>
|
||||
<version>${awssdk.version}</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/com.amazonaws/dynamodb-lock-client -->
|
||||
<dependency>
|
||||
<groupId>com.amazonaws</groupId>
|
||||
<artifactId>dynamodb-lock-client</artifactId>
|
||||
<version>1.3.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>software.amazon.awssdk</groupId>
|
||||
<artifactId>cloudwatch</artifactId>
|
||||
|
|
@ -75,10 +90,52 @@
|
|||
<artifactId>netty-nio-client</artifactId>
|
||||
<version>${awssdk.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>software.amazon.awssdk</groupId>
|
||||
<artifactId>sdk-core</artifactId>
|
||||
<version>${awssdk.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>software.amazon.awssdk</groupId>
|
||||
<artifactId>aws-core</artifactId>
|
||||
<version>${awssdk.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>software.amazon.awssdk</groupId>
|
||||
<artifactId>arns</artifactId>
|
||||
<version>${awssdk.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>software.amazon.awssdk</groupId>
|
||||
<artifactId>regions</artifactId>
|
||||
<version>${awssdk.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>software.amazon.awssdk</groupId>
|
||||
<artifactId>utils</artifactId>
|
||||
<version>${awssdk.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>software.amazon.awssdk</groupId>
|
||||
<artifactId>http-client-spi</artifactId>
|
||||
<version>${awssdk.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>software.amazon.awssdk</groupId>
|
||||
<artifactId>dynamodb-enhanced</artifactId>
|
||||
<version>${awssdk.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>software.amazon.glue</groupId>
|
||||
<artifactId>schema-registry-serde</artifactId>
|
||||
<version>${gsr.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>com.amazonaws</groupId>
|
||||
<artifactId>aws-java-sdk-sts</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>software.amazon.glue</groupId>
|
||||
|
|
@ -88,28 +145,70 @@
|
|||
<dependency>
|
||||
<groupId>com.google.guava</groupId>
|
||||
<artifactId>guava</artifactId>
|
||||
<version>32.0.0-jre</version>
|
||||
<version>32.1.1-jre</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.google.protobuf</groupId>
|
||||
<artifactId>protobuf-java</artifactId>
|
||||
<version>3.21.12</version>
|
||||
<version>${protobuf.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-lang3</artifactId>
|
||||
<version>3.12.0</version>
|
||||
<version>3.14.0</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/commons-collections/commons-collections -->
|
||||
<dependency>
|
||||
<groupId>commons-collections</groupId>
|
||||
<artifactId>commons-collections</artifactId>
|
||||
<version>3.2.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-collections4</artifactId>
|
||||
<version>4.4</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.netty</groupId>
|
||||
<artifactId>netty-handler</artifactId>
|
||||
<version>4.1.118.Final</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.google.code.findbugs</groupId>
|
||||
<artifactId>jsr305</artifactId>
|
||||
<version>3.0.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-databind</artifactId>
|
||||
<version>2.12.7.1</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.reactivestreams</groupId>
|
||||
<artifactId>reactive-streams</artifactId>
|
||||
<version>1.0.4</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>software.amazon.awssdk</groupId>
|
||||
<artifactId>annotations</artifactId>
|
||||
<version>2.25.64</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
<version>${slf4j.version}</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/org.jetbrains/annotations -->
|
||||
<dependency>
|
||||
<groupId>org.jetbrains</groupId>
|
||||
<artifactId>annotations</artifactId>
|
||||
<version>26.0.1</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.reactivex.rxjava3</groupId>
|
||||
<artifactId>rxjava</artifactId>
|
||||
<version>3.1.6</version>
|
||||
<version>3.1.8</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
|
|
@ -120,39 +219,75 @@
|
|||
</dependency>
|
||||
|
||||
<!-- Test -->
|
||||
<dependency>
|
||||
<groupId>software.amazon.awssdk</groupId>
|
||||
<artifactId>sts</artifactId>
|
||||
<version>${awssdk.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>software.amazon.awssdk</groupId>
|
||||
<artifactId>auth</artifactId>
|
||||
<version>${awssdk.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<!-- TODO: Migrate all tests to Junit5 -->
|
||||
<dependency>
|
||||
<groupId>org.junit.jupiter</groupId>
|
||||
<artifactId>junit-jupiter-api</artifactId>
|
||||
<version>5.11.3</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<version>4.13.2</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.junit.jupiter/junit-jupiter-params -->
|
||||
<dependency>
|
||||
<groupId>org.mockito</groupId>
|
||||
<artifactId>mockito-all</artifactId>
|
||||
<version>1.10.19</version>
|
||||
<groupId>org.junit.jupiter</groupId>
|
||||
<artifactId>junit-jupiter-params</artifactId>
|
||||
<version>5.11.3</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<!-- Using older version to be compatible with Java 8 -->
|
||||
<dependency>
|
||||
<groupId>org.mockito</groupId>
|
||||
<artifactId>mockito-junit-jupiter</artifactId>
|
||||
<version>3.12.4</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.mockito</groupId>
|
||||
<artifactId>mockito-core</artifactId>
|
||||
<version>3.12.4</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.hamcrest</groupId>
|
||||
<artifactId>hamcrest-all</artifactId>
|
||||
<version>1.3</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
|
||||
<!--<dependency>-->
|
||||
<!--<groupId>com.amazonaws</groupId>-->
|
||||
<!--<artifactId>DynamoDBLocal</artifactId>-->
|
||||
<!--<version>1.11.86</version>-->
|
||||
<!--<scope>test</scope>-->
|
||||
<!--</dependency>-->
|
||||
|
||||
<dependency>
|
||||
<groupId>org.hamcrest</groupId>
|
||||
<artifactId>hamcrest-core</artifactId>
|
||||
<version>1.3</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<!-- Using older version to be compatible with Java 8 -->
|
||||
<!-- https://mvnrepository.com/artifact/com.amazonaws/DynamoDBLocal -->
|
||||
<dependency>
|
||||
<groupId>com.amazonaws</groupId>
|
||||
<artifactId>DynamoDBLocal</artifactId>
|
||||
<version>1.25.0</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>ch.qos.logback</groupId>
|
||||
<artifactId>logback-classic</artifactId>
|
||||
<version>1.3.0</version>
|
||||
<version>1.3.14</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
|
|
@ -178,15 +313,21 @@
|
|||
</developers>
|
||||
|
||||
<build>
|
||||
<extensions>
|
||||
<extension>
|
||||
<groupId>kr.motd.maven</groupId>
|
||||
<artifactId>os-maven-plugin</artifactId>
|
||||
<version>1.6.0</version>
|
||||
</extension>
|
||||
</extensions>
|
||||
<pluginManagement>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.11.0</version>
|
||||
<version>3.13.0</version>
|
||||
<configuration>
|
||||
<source>1.8</source>
|
||||
<target>1.8</target>
|
||||
<release>8</release>
|
||||
<encoding>UTF-8</encoding>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
|
@ -194,30 +335,41 @@
|
|||
</pluginManagement>
|
||||
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.xolstice.maven.plugins</groupId>
|
||||
<artifactId>protobuf-maven-plugin</artifactId>
|
||||
<version>0.6.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<goals>
|
||||
<goal>compile</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
<configuration>
|
||||
<protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-surefire-plugin</artifactId>
|
||||
<version>2.22.2</version>
|
||||
<version>3.2.5</version>
|
||||
<configuration>
|
||||
<skipTests>${skip.ut}</skipTests>
|
||||
<skipITs>${skipITs}</skipITs>
|
||||
<excludes>
|
||||
<exclude>**/*IntegrationTest.java</exclude>
|
||||
</excludes>
|
||||
<systemProperties>
|
||||
<property>
|
||||
<name>sqlite4java.library.path</name>
|
||||
<value>${sqlite4java.libpath}</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>awsProfile</name>
|
||||
<value>${awsProfile}</value>
|
||||
</property>
|
||||
</systemProperties>
|
||||
<systemPropertyVariables>
|
||||
<sqlite4java.library.path>${sqlite4java.libpath}</sqlite4java.library.path>
|
||||
<awsProfile>${awsProfile}</awsProfile>
|
||||
</systemPropertyVariables>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-failsafe-plugin</artifactId>
|
||||
<version>2.22.2</version>
|
||||
<version>3.2.5</version>
|
||||
<configuration>
|
||||
<includes>
|
||||
<include>**/*IntegrationTest.java</include>
|
||||
|
|
@ -303,7 +455,7 @@
|
|||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
<version>3.5.0</version>
|
||||
<version>3.7.0</version>
|
||||
<configuration>
|
||||
<excludePackageNames>com.amazonaws.services.kinesis.producer.protobuf</excludePackageNames>
|
||||
</configuration>
|
||||
|
|
@ -347,7 +499,7 @@
|
|||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-resources-plugin</artifactId>
|
||||
<version>3.3.0</version>
|
||||
<version>3.3.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>copy-dist</id>
|
||||
|
|
@ -369,6 +521,58 @@
|
|||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>com.diffplug.spotless</groupId>
|
||||
<artifactId>spotless-maven-plugin</artifactId>
|
||||
<version>2.30.0</version> <!--last version to support java 8-->
|
||||
<configuration>
|
||||
<java>
|
||||
<palantirJavaFormat />
|
||||
<importOrder>
|
||||
<order>java,,\#</order>
|
||||
</importOrder>
|
||||
</java>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
<phase>compile</phase>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>com.salesforce.servicelibs</groupId>
|
||||
<artifactId>proto-backwards-compatibility</artifactId>
|
||||
<version>1.0.7</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<goals>
|
||||
<goal>backwards-compatibility-check</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-dependency-plugin</artifactId>
|
||||
<version>3.1.2</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>analyze-dependencies</id>
|
||||
<phase>verify</phase>
|
||||
<goals>
|
||||
<goal>analyze-only</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<failOnWarning>true</failOnWarning>
|
||||
<!-- Ignore Runtime/Provided/Test/System scopes for unused dependency analysis. -->
|
||||
<ignoreNonCompile>true</ignoreNonCompile>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
|
||||
</build>
|
||||
|
|
|
|||
609
amazon-kinesis-client/scripts/KclMigrationTool.py
Normal file
609
amazon-kinesis-client/scripts/KclMigrationTool.py
Normal file
|
|
@ -0,0 +1,609 @@
|
|||
"""
|
||||
Copyright 2024 Amazon.com, Inc. or its affiliates.
|
||||
Licensed under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import time
|
||||
|
||||
from enum import Enum
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
# DynamoDB table suffixes
|
||||
DEFAULT_COORDINATOR_STATE_TABLE_SUFFIX = "-CoordinatorState"
|
||||
DEFAULT_WORKER_METRICS_TABLE_SUFFIX = "-WorkerMetricStats"
|
||||
|
||||
# DynamoDB attribute names and values
|
||||
CLIENT_VERSION_ATTR = 'cv'
|
||||
TIMESTAMP_ATTR = 'mts'
|
||||
MODIFIED_BY_ATTR = 'mb'
|
||||
HISTORY_ATTR = 'h'
|
||||
MIGRATION_KEY = "Migration3.0"
|
||||
|
||||
# GSI constants
|
||||
GSI_NAME = 'LeaseOwnerToLeaseKeyIndex'
|
||||
GSI_DELETION_WAIT_TIME_SECONDS = 120
|
||||
|
||||
config = Config(
|
||||
retries = {
|
||||
'max_attempts': 10,
|
||||
'mode': 'standard'
|
||||
}
|
||||
)
|
||||
|
||||
class KclClientVersion(Enum):
|
||||
VERSION_2X = "CLIENT_VERSION_2X"
|
||||
UPGRADE_FROM_2X = "CLIENT_VERSION_UPGRADE_FROM_2X"
|
||||
VERSION_3X_WITH_ROLLBACK = "CLIENT_VERSION_3X_WITH_ROLLBACK"
|
||||
VERSION_3X = "CLIENT_VERSION_3X"
|
||||
|
||||
def __str__(self):
|
||||
return self.value
|
||||
|
||||
|
||||
def get_time_in_millis():
|
||||
return str(round(time.time() * 1000))
|
||||
|
||||
|
||||
def is_valid_version(version, mode):
|
||||
"""
|
||||
Validate if the given version is valid for the specified mode
|
||||
|
||||
:param version: The KCL client version to validate
|
||||
:param mode: Either 'rollback' or 'rollforward'
|
||||
:return: True if the version is valid for the given mode, False otherwise
|
||||
"""
|
||||
if mode == 'rollback':
|
||||
if version == KclClientVersion.VERSION_2X.value:
|
||||
print("Your KCL application already runs in a mode compatible with KCL 2.x. You can deploy the code with the previous KCL version if you still experience an issue.")
|
||||
return True
|
||||
if version in [KclClientVersion.UPGRADE_FROM_2X.value,
|
||||
KclClientVersion.VERSION_3X_WITH_ROLLBACK.value]:
|
||||
return True
|
||||
if version == KclClientVersion.VERSION_3X.value:
|
||||
print("Cannot roll back the KCL application."
|
||||
" It is not in a state that supports rollback.")
|
||||
return False
|
||||
print("Migration to KCL 3.0 not in progress or application_name / coordinator_state_table_name is incorrect."
|
||||
" Please double check and run again with correct arguments.")
|
||||
return False
|
||||
|
||||
if mode == 'rollforward':
|
||||
if version == KclClientVersion.VERSION_2X.value:
|
||||
return True
|
||||
if version in [KclClientVersion.UPGRADE_FROM_2X.value,
|
||||
KclClientVersion.VERSION_3X_WITH_ROLLBACK.value]:
|
||||
print("Cannot roll-forward application. It is not in a rolled back state.")
|
||||
return False
|
||||
if version == KclClientVersion.VERSION_3X.value:
|
||||
print("Cannot roll-forward the KCL application."
|
||||
" Application has already migrated.")
|
||||
return False
|
||||
print("Cannot roll-forward because migration to KCL 3.0 is not in progress or application_name"
|
||||
" / coordinator_state_table_name is incorrect. Please double check and run again with correct arguments.")
|
||||
return False
|
||||
print(f"Invalid mode: {mode}. Mode must be either 'rollback' or 'rollforward'.")
|
||||
return False
|
||||
|
||||
|
||||
def handle_get_item_client_error(e, operation, table_name):
|
||||
"""
|
||||
Handle ClientError exceptions raised by get_item on given DynamoDB table
|
||||
|
||||
:param e: The ClientError exception object
|
||||
:param operation: Rollback or Roll-forward for logging the errors
|
||||
:param table_name: The name of the DynamoDB table where the error occurred
|
||||
"""
|
||||
error_code = e.response['Error']['Code']
|
||||
error_message = e.response['Error']['Message']
|
||||
print(f"{operation} could not be performed.")
|
||||
if error_code == 'ProvisionedThroughputExceededException':
|
||||
print(f"Throughput exceeded even after retries: {error_message}")
|
||||
else:
|
||||
print(f"Unexpected client error occurred: {error_code} - {error_message}")
|
||||
print("Please resolve the issue and run the KclMigrationTool again.")
|
||||
|
||||
|
||||
def table_exists(dynamodb_client, table_name):
|
||||
"""
|
||||
Check if a DynamoDB table exists.
|
||||
|
||||
:param dynamodb_client: Boto3 DynamoDB client
|
||||
:param table_name: Name of the DynamoDB table to check
|
||||
:return: True if the table exists, False otherwise
|
||||
"""
|
||||
try:
|
||||
dynamodb_client.describe_table(TableName=table_name)
|
||||
return True
|
||||
except ClientError as e:
|
||||
if e.response['Error']['Code'] == 'ResourceNotFoundException':
|
||||
print(f"Table '{table_name}' does not exist.")
|
||||
return False
|
||||
print(f"An error occurred while checking table '{table_name}': {e}.")
|
||||
return False
|
||||
|
||||
|
||||
def validate_tables(dynamodb_client, operation, coordinator_state_table_name, lease_table_name=None):
|
||||
"""
|
||||
Validate the existence of DynamoDB tables required for KCL operations
|
||||
|
||||
:param dynamodb_client: A boto3 DynamoDB client object
|
||||
:param operation: Rollback or Roll-forward for logging
|
||||
:param coordinator_state_table_name: Name of the coordinator state table
|
||||
:param lease_table_name: Name of the DynamoDB KCL lease table (optional)
|
||||
:return: True if all required tables exist, False otherwise
|
||||
"""
|
||||
if lease_table_name and not table_exists(dynamodb_client, lease_table_name):
|
||||
print(
|
||||
f"{operation} failed. Could not find a KCL Application DDB lease table "
|
||||
f"with name {lease_table_name}. Please pass in the correct application_name "
|
||||
"and/or lease_table_name that matches your KCL application configuration."
|
||||
)
|
||||
return False
|
||||
|
||||
if not table_exists(dynamodb_client, coordinator_state_table_name):
|
||||
print(
|
||||
f"{operation} failed. Could not find a coordinator state table "
|
||||
f"{coordinator_state_table_name}. Please pass in the correct application_name or"
|
||||
f" coordinator_state_table_name that matches your KCL application configuration."
|
||||
)
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def add_current_state_to_history(item, max_history=10):
|
||||
"""
|
||||
Adds the current state of a DynamoDB item to its history attribute.
|
||||
Creates a new history entry from the current value and maintains a capped history list.
|
||||
|
||||
:param item: DynamoDB item to add history to
|
||||
:param max_history: Maximum number of history entries to maintain (default: 10)
|
||||
:return: Updated history attribute as a DynamoDB-formatted dictionary
|
||||
"""
|
||||
# Extract current values
|
||||
current_version = item.get(CLIENT_VERSION_ATTR, {}).get('S', 'Unknown')
|
||||
current_modified_by = item.get(MODIFIED_BY_ATTR, {}).get('S', 'Unknown')
|
||||
current_time_in_millis = (
|
||||
item.get(TIMESTAMP_ATTR, {}).get('N', get_time_in_millis())
|
||||
)
|
||||
|
||||
# Create new history entry
|
||||
new_entry = {
|
||||
'M': {
|
||||
CLIENT_VERSION_ATTR: {'S': current_version},
|
||||
MODIFIED_BY_ATTR: {'S': current_modified_by},
|
||||
TIMESTAMP_ATTR: {'N': current_time_in_millis}
|
||||
}
|
||||
}
|
||||
|
||||
# Get existing history or create new if doesn't exist
|
||||
history_dict = item.get(f'{HISTORY_ATTR}', {'L': []})
|
||||
history_list = history_dict['L']
|
||||
|
||||
# Add new entry to the beginning of the list, capping at max_history
|
||||
history_list.insert(0, new_entry)
|
||||
history_list = history_list[:max_history]
|
||||
|
||||
return history_dict
|
||||
|
||||
|
||||
def get_current_state(dynamodb_client, table_name):
|
||||
"""
|
||||
Retrieve the current state from the DynamoDB table and prepare history update.
|
||||
Fetches the current item from the specified DynamoDB table,
|
||||
extracts the initial client version, and creates a new history entry.
|
||||
|
||||
:param dynamodb_client: Boto3 DynamoDB client
|
||||
:param table_name: Name of the DynamoDB table to query
|
||||
:return: A tuple containing:
|
||||
- initial_version (str): The current client version, or 'Unknown' if not found
|
||||
- new_history (dict): Updated history including the current state
|
||||
"""
|
||||
response = dynamodb_client.get_item(
|
||||
TableName=table_name,
|
||||
Key={'key': {'S': MIGRATION_KEY}}
|
||||
)
|
||||
item = response.get('Item', {})
|
||||
initial_version = item.get(CLIENT_VERSION_ATTR, {}).get('S', 'Unknown')
|
||||
new_history = add_current_state_to_history(item)
|
||||
return initial_version, new_history
|
||||
|
||||
|
||||
def rollback_client_version(dynamodb_client, table_name, history):
|
||||
"""
|
||||
Update the client version in the coordinator state table to initiate rollback.
|
||||
|
||||
:param dynamodb_client: Boto3 DynamoDB client
|
||||
:param table_name: Name of the coordinator state DDB table
|
||||
:param history: Updated history attribute as a DynamoDB-formatted dictionary
|
||||
:return: A tuple containing:
|
||||
- success (bool): True if client version was successfully updated, False otherwise
|
||||
- previous_version (str): The version that was replaced, or None if update failed
|
||||
"""
|
||||
try:
|
||||
print(f"Rolling back client version in table '{table_name}'...")
|
||||
update_response = dynamodb_client.update_item(
|
||||
TableName=table_name,
|
||||
Key={'key': {'S': MIGRATION_KEY}},
|
||||
UpdateExpression=(
|
||||
f"SET {CLIENT_VERSION_ATTR} = :rollback_client_version, "
|
||||
f"{TIMESTAMP_ATTR} = :updated_at, "
|
||||
f"{MODIFIED_BY_ATTR} = :modifier, "
|
||||
f"{HISTORY_ATTR} = :history"
|
||||
),
|
||||
ConditionExpression=(
|
||||
f"{CLIENT_VERSION_ATTR} IN ("
|
||||
":upgrade_from_2x_client_version, "
|
||||
":3x_with_rollback_client_version)"
|
||||
),
|
||||
ExpressionAttributeValues={
|
||||
':rollback_client_version': {'S': KclClientVersion.VERSION_2X.value},
|
||||
':updated_at': {'N': get_time_in_millis()},
|
||||
':modifier': {'S': 'KclMigrationTool-rollback'},
|
||||
':history': history,
|
||||
':upgrade_from_2x_client_version': (
|
||||
{'S': KclClientVersion.UPGRADE_FROM_2X.value}
|
||||
),
|
||||
':3x_with_rollback_client_version': (
|
||||
{'S': KclClientVersion.VERSION_3X_WITH_ROLLBACK.value}
|
||||
),
|
||||
},
|
||||
ReturnValues='UPDATED_OLD'
|
||||
)
|
||||
replaced_item = update_response.get('Attributes', {})
|
||||
replaced_version = replaced_item.get('cv', {}).get('S', '')
|
||||
return True, replaced_version
|
||||
except ClientError as e:
|
||||
if e.response['Error']['Code'] == 'ConditionalCheckFailedException':
|
||||
print("Unable to rollback, as application is not in a state that allows rollback."
|
||||
"Ensure that the given application_name or coordinator_state_table_name is correct and"
|
||||
" you have followed all prior migration steps.")
|
||||
else:
|
||||
print(f"An unexpected error occurred while rolling back: {str(e)}"
|
||||
"Please resolve and run this migration script again.")
|
||||
return False, None
|
||||
|
||||
|
||||
def rollfoward_client_version(dynamodb_client, table_name, history):
|
||||
"""
|
||||
Update the client version in the coordinator state table to initiate roll-forward
|
||||
conditionally if application is currently in rolled back state.
|
||||
|
||||
:param dynamodb_client: Boto3 DynamoDB client
|
||||
:param table_name: Name of the coordinator state DDB table
|
||||
:param history: Updated history attribute as a DynamoDB-formatted dictionary
|
||||
:return: True if client version was successfully updated, False otherwise
|
||||
"""
|
||||
try:
|
||||
# Conditionally update client version
|
||||
dynamodb_client.update_item(
|
||||
TableName=table_name,
|
||||
Key={'key': {'S': MIGRATION_KEY}},
|
||||
UpdateExpression= (
|
||||
f"SET {CLIENT_VERSION_ATTR} = :rollforward_version, "
|
||||
f"{TIMESTAMP_ATTR} = :updated_at, "
|
||||
f"{MODIFIED_BY_ATTR} = :modifier, "
|
||||
f"{HISTORY_ATTR} = :new_history"
|
||||
),
|
||||
ConditionExpression=f"{CLIENT_VERSION_ATTR} = :kcl_2x_version",
|
||||
ExpressionAttributeValues={
|
||||
':rollforward_version': {'S': KclClientVersion.UPGRADE_FROM_2X.value},
|
||||
':updated_at': {'N': get_time_in_millis()},
|
||||
':modifier': {'S': 'KclMigrationTool-rollforward'},
|
||||
':new_history': history,
|
||||
':kcl_2x_version': {'S': KclClientVersion.VERSION_2X.value},
|
||||
}
|
||||
)
|
||||
print("Roll-forward has been initiated. KCL application will monitor for 3.0 readiness and"
|
||||
" automatically switch to 3.0 functionality when readiness criteria have been met.")
|
||||
except ClientError as e:
|
||||
if e.response['Error']['Code'] == 'ConditionalCheckFailedException':
|
||||
print("Unable to roll-forward because application is not in rolled back state."
|
||||
" Ensure that the given application_name or coordinator_state_table_name is correct"
|
||||
" and you have followed all prior migration steps.")
|
||||
else:
|
||||
print(f"Unable to roll-forward due to error: {str(e)}. "
|
||||
"Please resolve and run this migration script again.")
|
||||
except Exception as e:
|
||||
print(f"Unable to roll-forward due to error: {str(e)}. "
|
||||
"Please resolve and run this migration script again.")
|
||||
|
||||
|
||||
def delete_gsi_if_exists(dynamodb_client, table_name):
|
||||
"""
|
||||
Deletes GSI on given lease table if it exists.
|
||||
|
||||
:param dynamodb_client: Boto3 DynamoDB client
|
||||
:param table_name: Name of lease table to remove GSI from
|
||||
"""
|
||||
try:
|
||||
gsi_present = False
|
||||
response = dynamodb_client.describe_table(TableName=table_name)
|
||||
if 'GlobalSecondaryIndexes' in response['Table']:
|
||||
gsi_list = response['Table']['GlobalSecondaryIndexes']
|
||||
for gsi in gsi_list:
|
||||
if gsi['IndexName'] == GSI_NAME:
|
||||
gsi_present = True
|
||||
break
|
||||
|
||||
if not gsi_present:
|
||||
print(f"GSI {GSI_NAME} is not present on lease table {table_name}. It may already be successfully"
|
||||
" deleted. Or if lease table name is incorrect, please re-run the KclMigrationTool with correct"
|
||||
" application_name or lease_table_name.")
|
||||
return
|
||||
except ClientError as e:
|
||||
if e.response['Error']['Code'] == 'ResourceNotFoundException':
|
||||
print(f"Lease table {table_name} does not exist, please check application_name or lease_table_name"
|
||||
" configuration and try again.")
|
||||
return
|
||||
else:
|
||||
print(f"An unexpected error occurred while checking if GSI {GSI_NAME} exists"
|
||||
f" on lease table {table_name}: {str(e)}. Please rectify the error and try again.")
|
||||
return
|
||||
|
||||
print(f"Deleting GSI '{GSI_NAME}' from table '{table_name}'...")
|
||||
try:
|
||||
dynamodb_client.update_table(
|
||||
TableName=table_name,
|
||||
GlobalSecondaryIndexUpdates=[
|
||||
{
|
||||
'Delete': {
|
||||
'IndexName': GSI_NAME
|
||||
}
|
||||
}
|
||||
]
|
||||
)
|
||||
except ClientError as e:
|
||||
if e.response['Error']['Code'] == 'ResourceNotFoundException':
|
||||
print(f"{GSI_NAME} not found or table '{table_name}' not found.")
|
||||
elif e.response['Error']['Code'] == 'ResourceInUseException':
|
||||
print(f"Unable to delete GSI: '{table_name}' is currently being modified.")
|
||||
except Exception as e:
|
||||
print(f"An unexpected error occurred while deleting GSI {GSI_NAME} on lease table {table_name}: {str(e)}."
|
||||
" Please manually confirm the GSI is removed from the lease table, or"
|
||||
" resolve the error and rerun the migration script.")
|
||||
|
||||
|
||||
def delete_worker_metrics_table_if_exists(dynamodb_client, worker_metrics_table_name):
|
||||
"""
|
||||
Deletes worker metrics table based on application name, if it exists.
|
||||
|
||||
:param dynamodb_client: Boto3 DynamoDB client
|
||||
:param worker_metrics_table_name: Name of the DynamoDB worker metrics table
|
||||
"""
|
||||
try:
|
||||
dynamodb_client.describe_table(TableName=worker_metrics_table_name)
|
||||
except ClientError as e:
|
||||
if e.response['Error']['Code'] == 'ResourceNotFoundException':
|
||||
print(f"Worker metrics table {worker_metrics_table_name} does not exist."
|
||||
" It may already be successfully deleted. Please check that the application_name"
|
||||
" or worker_metrics_table_name is correct. If not, correct this and rerun the migration script.")
|
||||
return
|
||||
else:
|
||||
print(f"An unexpected error occurred when checking if {worker_metrics_table_name} table exists: {str(e)}."
|
||||
" Please manually confirm the table is deleted, or resolve the error"
|
||||
" and rerun the migration script.")
|
||||
return
|
||||
|
||||
print(f"Deleting worker metrics table {worker_metrics_table_name}...")
|
||||
try:
|
||||
dynamodb_client.delete_table(TableName=worker_metrics_table_name)
|
||||
except ClientError as e:
|
||||
if e.response['Error']['Code'] == 'AccessDeniedException':
|
||||
print(f"No permissions to delete table {worker_metrics_table_name}. Please manually delete it if you"
|
||||
" want to avoid any charges until you are ready to rollforward with migration.")
|
||||
else:
|
||||
print(f"An unexpected client error occurred while deleting worker metrics table: {str(e)}."
|
||||
" Please manually confirm the table is deleted, or resolve the error"
|
||||
" and rerun the migration script.")
|
||||
except Exception as e:
|
||||
print(f"An unexpected error occurred while deleting worker metrics table: {str(e)}."
|
||||
" Please manually confirm the table is deleted, or resolve the error"
|
||||
" and rerun the migration script.")
|
||||
|
||||
|
||||
def perform_rollback(dynamodb_client, lease_table_name, coordinator_state_table_name, worker_metrics_table_name):
|
||||
"""
|
||||
Perform KCL 3.0 migration rollback by updating MigrationState for the KCL application.
|
||||
Rolls client version back, removes GSI from lease table, deletes worker metrics table.
|
||||
|
||||
:param dynamodb_client: Boto3 DynamoDB client
|
||||
:param coordinator_state_table_name: Name of the DynamoDB coordinator state table
|
||||
:param coordinator_state_table_name: Name of the DynamoDB coordinator state table
|
||||
:param worker_metrics_table_name: Name of the DynamoDB worker metrics table
|
||||
"""
|
||||
if not validate_tables(dynamodb_client, "Rollback", coordinator_state_table_name, lease_table_name):
|
||||
return
|
||||
|
||||
try:
|
||||
initial_version, new_history = get_current_state(dynamodb_client,
|
||||
coordinator_state_table_name)
|
||||
except ClientError as e:
|
||||
handle_get_item_client_error(e, "Rollback", coordinator_state_table_name)
|
||||
return
|
||||
|
||||
if not is_valid_version(version=initial_version, mode='rollback'):
|
||||
return
|
||||
|
||||
# 1. Rollback client version
|
||||
if initial_version != KclClientVersion.VERSION_2X.value:
|
||||
rollback_succeeded, initial_version = rollback_client_version(
|
||||
dynamodb_client, coordinator_state_table_name, new_history
|
||||
)
|
||||
if not rollback_succeeded:
|
||||
return
|
||||
|
||||
print(f"Waiting for {GSI_DELETION_WAIT_TIME_SECONDS} seconds before cleaning up KCL 3.0 resources after rollback...")
|
||||
time.sleep(GSI_DELETION_WAIT_TIME_SECONDS)
|
||||
|
||||
# 2. Delete the GSI
|
||||
delete_gsi_if_exists(dynamodb_client, lease_table_name)
|
||||
|
||||
# 3. Delete worker metrics table
|
||||
delete_worker_metrics_table_if_exists(dynamodb_client, worker_metrics_table_name)
|
||||
|
||||
# Log success
|
||||
if initial_version == KclClientVersion.UPGRADE_FROM_2X.value:
|
||||
print("\nRollback completed. Your application was running 2x compatible functionality.")
|
||||
print("Please rollback to your previous application binaries by deploying the code with your previous KCL version.")
|
||||
elif initial_version == KclClientVersion.VERSION_3X_WITH_ROLLBACK.value:
|
||||
print("\nRollback completed. Your KCL Application was running 3x functionality and will rollback to 2x compatible functionality.")
|
||||
print("If you don't see mitigation after a short period of time,"
|
||||
" please rollback to your previous application binaries by deploying the code with your previous KCL version.")
|
||||
elif initial_version == KclClientVersion.VERSION_2X.value:
|
||||
print("\nApplication was already rolled back. Any KCLv3 resources that could be deleted were cleaned up"
|
||||
" to avoid charges until the application can be rolled forward with migration.")
|
||||
|
||||
|
||||
def perform_rollforward(dynamodb_client, coordinator_state_table_name):
|
||||
"""
|
||||
Perform KCL 3.0 migration roll-forward by updating MigrationState for the KCL application
|
||||
|
||||
:param dynamodb_client: Boto3 DynamoDB client
|
||||
:param coordinator_state_table_name: Name of the DynamoDB table
|
||||
"""
|
||||
if not validate_tables(dynamodb_client, "Roll-forward", coordinator_state_table_name):
|
||||
return
|
||||
|
||||
try:
|
||||
initial_version, new_history = get_current_state(dynamodb_client,
|
||||
coordinator_state_table_name)
|
||||
except ClientError as e:
|
||||
handle_get_item_client_error(e, "Roll-forward", coordinator_state_table_name)
|
||||
return
|
||||
|
||||
if not is_valid_version(version=initial_version, mode='rollforward'):
|
||||
return
|
||||
|
||||
rollfoward_client_version(dynamodb_client, coordinator_state_table_name, new_history)
|
||||
|
||||
|
||||
def run_kcl_migration(mode, lease_table_name, coordinator_state_table_name, worker_metrics_table_name):
|
||||
"""
|
||||
Update the MigrationState in CoordinatorState DDB Table
|
||||
|
||||
:param mode: Either 'rollback' or 'rollforward'
|
||||
:param lease_table_name: Name of the DynamoDB KCL lease table
|
||||
:param coordinator_state_table_name: Name of the DynamoDB coordinator state table
|
||||
:param worker_metrics_table_name: Name of the DynamoDB worker metrics table
|
||||
"""
|
||||
dynamodb_client = boto3.client('dynamodb', config=config)
|
||||
|
||||
if mode == "rollback":
|
||||
perform_rollback(
|
||||
dynamodb_client,
|
||||
lease_table_name,
|
||||
coordinator_state_table_name,
|
||||
worker_metrics_table_name
|
||||
)
|
||||
elif mode == "rollforward":
|
||||
perform_rollforward(dynamodb_client, coordinator_state_table_name)
|
||||
else:
|
||||
print(f"Invalid mode: {mode}. Please use 'rollback' or 'rollforward'.")
|
||||
|
||||
|
||||
def validate_args(args):
|
||||
if args.mode == 'rollforward':
|
||||
if not (args.application_name or args.coordinator_state_table_name):
|
||||
raise ValueError(
|
||||
"For rollforward mode, either application_name or "
|
||||
"coordinator_state_table_name must be provided."
|
||||
)
|
||||
else:
|
||||
if args.application_name:
|
||||
return
|
||||
|
||||
if not (args.lease_table_name and
|
||||
args.coordinator_state_table_name and
|
||||
args.worker_metrics_table_name):
|
||||
raise ValueError(
|
||||
"For rollback mode, either application_name or all three table names "
|
||||
"(lease_table_name, coordinator_state_table_name, and "
|
||||
"worker_metrics_table_name) must be provided."
|
||||
)
|
||||
|
||||
def process_table_names(args):
|
||||
"""
|
||||
Process command line arguments to determine table names based on mode.
|
||||
Args:
|
||||
args: Parsed command line arguments
|
||||
Returns:
|
||||
tuple: (mode, lease_table_name, coordinator_state_table_name, worker_metrics_table_name)
|
||||
"""
|
||||
mode_input = args.mode
|
||||
application_name_input = args.application_name
|
||||
|
||||
coordinator_state_table_name_input = (args.coordinator_state_table_name or
|
||||
application_name_input + DEFAULT_COORDINATOR_STATE_TABLE_SUFFIX)
|
||||
lease_table_name_input = None
|
||||
worker_metrics_table_name_input = None
|
||||
|
||||
if mode_input == "rollback":
|
||||
lease_table_name_input = args.lease_table_name or application_name_input
|
||||
worker_metrics_table_name_input = (args.worker_metrics_table_name or
|
||||
application_name_input + DEFAULT_WORKER_METRICS_TABLE_SUFFIX)
|
||||
|
||||
return (mode_input,
|
||||
lease_table_name_input,
|
||||
coordinator_state_table_name_input,
|
||||
worker_metrics_table_name_input)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(
|
||||
description=
|
||||
"""
|
||||
KCL Migration Tool
|
||||
This tool facilitates the migration and rollback processes for Amazon KCLv3 applications.
|
||||
|
||||
Before running this tool:
|
||||
1. Ensure you have the necessary AWS permissions configured to access and modify the following:
|
||||
- KCL application DynamoDB tables (lease table and coordinator state table)
|
||||
|
||||
2. Verify that your AWS credentials are properly set up in your environment or AWS config file.
|
||||
|
||||
3. Confirm that you have the correct KCL application name and lease table name (if configured in KCL).
|
||||
|
||||
Usage:
|
||||
This tool supports two main operations: rollforward (upgrade) and rollback.
|
||||
For detailed usage instructions, use the -h or --help option.
|
||||
""",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument("--mode", choices=['rollback', 'rollforward'], required=True,
|
||||
help="Mode of operation: rollback or rollforward")
|
||||
parser.add_argument("--application_name",
|
||||
help="Name of the KCL application. This must match the application name "
|
||||
"used in the KCL Library configurations.")
|
||||
parser.add_argument("--lease_table_name",
|
||||
help="Name of the DynamoDB lease table (defaults to applicationName)."
|
||||
" If LeaseTable name was specified for the application as part of "
|
||||
"the KCL configurations, the same name must be passed here.")
|
||||
parser.add_argument("--coordinator_state_table_name",
|
||||
help="Name of the DynamoDB coordinator state table "
|
||||
"(defaults to applicationName-CoordinatorState)."
|
||||
" If coordinator state table name was specified for the application "
|
||||
"as part of the KCL configurations, the same name must be passed here.")
|
||||
parser.add_argument("--worker_metrics_table_name",
|
||||
help="Name of the DynamoDB worker metrics table "
|
||||
"(defaults to applicationName-WorkerMetricStats)."
|
||||
" If worker metrics table name was specified for the application "
|
||||
"as part of the KCL configurations, the same name must be passed here.")
|
||||
parser.add_argument("--region", required=True,
|
||||
help="AWS Region where your KCL application exists")
|
||||
args = parser.parse_args()
|
||||
validate_args(args)
|
||||
config.region_name = args.region
|
||||
run_kcl_migration(*process_table_names(args))
|
||||
|
|
@ -19,8 +19,9 @@ import java.lang.annotation.Retention;
|
|||
import java.lang.annotation.RetentionPolicy;
|
||||
|
||||
/**
|
||||
* Any class/method/variable marked with this annotation is subject to breaking changes between minor releases.
|
||||
* Marker interface for 'internal' APIs that should not be used outside the core module.
|
||||
* Breaking changes can and will be introduced to elements marked as KinesisClientInternalApi.
|
||||
* Users of the KCL should not depend on any packages, types, fields, constructors, or methods with this annotation.
|
||||
*/
|
||||
@Retention(RetentionPolicy.CLASS)
|
||||
public @interface KinesisClientInternalApi {
|
||||
}
|
||||
public @interface KinesisClientInternalApi {}
|
||||
|
|
|
|||
|
|
@ -40,7 +40,10 @@ public class Checkpoint {
|
|||
* @param pendingCheckpoint the pending checkpoint sequence number - can be null.
|
||||
* @param pendingCheckpointState the pending checkpoint state - can be null.
|
||||
*/
|
||||
public Checkpoint(final ExtendedSequenceNumber checkpoint, final ExtendedSequenceNumber pendingCheckpoint, byte[] pendingCheckpointState) {
|
||||
public Checkpoint(
|
||||
final ExtendedSequenceNumber checkpoint,
|
||||
final ExtendedSequenceNumber pendingCheckpoint,
|
||||
byte[] pendingCheckpointState) {
|
||||
if (checkpoint == null || checkpoint.sequenceNumber().isEmpty()) {
|
||||
throw new IllegalArgumentException("Checkpoint cannot be null or empty");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@
|
|||
|
||||
package software.amazon.kinesis.checkpoint;
|
||||
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.experimental.Accessors;
|
||||
import software.amazon.kinesis.checkpoint.dynamodb.DynamoDBCheckpointFactory;
|
||||
|
|
|
|||
|
|
@ -63,6 +63,4 @@ public class DoesNothingPreparedCheckpointer implements PreparedCheckpointer {
|
|||
IllegalArgumentException {
|
||||
// This method does nothing
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -20,10 +20,9 @@ import java.util.Collections;
|
|||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.experimental.Accessors;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
/**
|
||||
* This supports extracting the shardId from a sequence number.
|
||||
|
|
@ -98,11 +97,15 @@ public class SequenceNumberValidator {
|
|||
}
|
||||
}
|
||||
|
||||
private static final List<SequenceNumberReader> SEQUENCE_NUMBER_READERS = Collections
|
||||
.singletonList(new V2SequenceNumberReader());
|
||||
private static final List<SequenceNumberReader> SEQUENCE_NUMBER_READERS =
|
||||
Collections.singletonList(new V2SequenceNumberReader());
|
||||
|
||||
private Optional<SequenceNumberComponents> retrieveComponentsFor(String sequenceNumber) {
|
||||
return SEQUENCE_NUMBER_READERS.stream().map(r -> r.read(sequenceNumber)).filter(Optional::isPresent).map(Optional::get).findFirst();
|
||||
return SEQUENCE_NUMBER_READERS.stream()
|
||||
.map(r -> r.read(sequenceNumber))
|
||||
.filter(Optional::isPresent)
|
||||
.map(Optional::get)
|
||||
.findFirst();
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -184,5 +187,4 @@ public class SequenceNumberValidator {
|
|||
public Optional<Boolean> validateSequenceNumberForShard(String sequenceNumber, String shardId) {
|
||||
return shardIdFor(sequenceNumber).map(s -> StringUtils.equalsIgnoreCase(s, shardId));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -38,8 +38,8 @@ public class ShardPreparedCheckpointer implements PreparedCheckpointer {
|
|||
* @param pendingCheckpointSequenceNumber sequence number to checkpoint at
|
||||
* @param checkpointer checkpointer to use
|
||||
*/
|
||||
public ShardPreparedCheckpointer(ExtendedSequenceNumber pendingCheckpointSequenceNumber,
|
||||
RecordProcessorCheckpointer checkpointer) {
|
||||
public ShardPreparedCheckpointer(
|
||||
ExtendedSequenceNumber pendingCheckpointSequenceNumber, RecordProcessorCheckpointer checkpointer) {
|
||||
this.pendingCheckpointSequenceNumber = pendingCheckpointSequenceNumber;
|
||||
this.checkpointer = checkpointer;
|
||||
}
|
||||
|
|
@ -59,7 +59,7 @@ public class ShardPreparedCheckpointer implements PreparedCheckpointer {
|
|||
public void checkpoint()
|
||||
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException,
|
||||
IllegalArgumentException {
|
||||
checkpointer.checkpoint(pendingCheckpointSequenceNumber.sequenceNumber(),
|
||||
pendingCheckpointSequenceNumber.subSequenceNumber());
|
||||
checkpointer.checkpoint(
|
||||
pendingCheckpointSequenceNumber.sequenceNumber(), pendingCheckpointSequenceNumber.subSequenceNumber());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -41,16 +41,22 @@ import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber;
|
|||
public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpointer {
|
||||
@NonNull
|
||||
private final ShardInfo shardInfo;
|
||||
|
||||
@NonNull
|
||||
@Getter @Accessors(fluent = true)
|
||||
@Getter
|
||||
@Accessors(fluent = true)
|
||||
private final Checkpointer checkpointer;
|
||||
|
||||
// Set to the last value set via checkpoint().
|
||||
// Sample use: verify application shutdown() invoked checkpoint() at the end of a shard.
|
||||
@Getter @Accessors(fluent = true)
|
||||
@Getter
|
||||
@Accessors(fluent = true)
|
||||
private ExtendedSequenceNumber lastCheckpointValue;
|
||||
@Getter @Accessors(fluent = true)
|
||||
|
||||
@Getter
|
||||
@Accessors(fluent = true)
|
||||
private ExtendedSequenceNumber largestPermittedCheckpointValue;
|
||||
|
||||
private ExtendedSequenceNumber sequenceNumberAtShardEnd;
|
||||
|
||||
/**
|
||||
|
|
@ -60,8 +66,11 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
|
|||
public synchronized void checkpoint()
|
||||
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
|
||||
if (log.isDebugEnabled()) {
|
||||
log.debug("Checkpointing {}, token {} at largest permitted value {}", ShardInfo.getLeaseKey(shardInfo),
|
||||
shardInfo.concurrencyToken(), this.largestPermittedCheckpointValue);
|
||||
log.debug(
|
||||
"Checkpointing {}, token {} at largest permitted value {}",
|
||||
ShardInfo.getLeaseKey(shardInfo),
|
||||
shardInfo.concurrencyToken(),
|
||||
this.largestPermittedCheckpointValue);
|
||||
}
|
||||
advancePosition(this.largestPermittedCheckpointValue);
|
||||
}
|
||||
|
|
@ -103,8 +112,8 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
|
|||
IllegalArgumentException {
|
||||
|
||||
if (subSequenceNumber < 0) {
|
||||
throw new IllegalArgumentException("Could not checkpoint at invalid, negative subsequence number "
|
||||
+ subSequenceNumber);
|
||||
throw new IllegalArgumentException(
|
||||
"Could not checkpoint at invalid, negative subsequence number " + subSequenceNumber);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -116,8 +125,11 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
|
|||
&& newCheckpoint.compareTo(largestPermittedCheckpointValue) <= 0) {
|
||||
|
||||
if (log.isDebugEnabled()) {
|
||||
log.debug("Checkpointing {}, token {} at specific extended sequence number {}", ShardInfo.getLeaseKey(shardInfo),
|
||||
shardInfo.concurrencyToken(), newCheckpoint);
|
||||
log.debug(
|
||||
"Checkpointing {}, token {} at specific extended sequence number {}",
|
||||
ShardInfo.getLeaseKey(shardInfo),
|
||||
shardInfo.concurrencyToken(),
|
||||
newCheckpoint);
|
||||
}
|
||||
this.advancePosition(newCheckpoint);
|
||||
} else {
|
||||
|
|
@ -190,7 +202,8 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
|
|||
*/
|
||||
@Override
|
||||
public PreparedCheckpointer prepareCheckpoint(String sequenceNumber, byte[] applicationState)
|
||||
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException, IllegalArgumentException {
|
||||
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException,
|
||||
IllegalArgumentException {
|
||||
return prepareCheckpoint(sequenceNumber, 0, applicationState);
|
||||
}
|
||||
|
||||
|
|
@ -207,11 +220,13 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
|
|||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public PreparedCheckpointer prepareCheckpoint(String sequenceNumber, long subSequenceNumber, byte[] applicationState)
|
||||
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException, IllegalArgumentException {
|
||||
public PreparedCheckpointer prepareCheckpoint(
|
||||
String sequenceNumber, long subSequenceNumber, byte[] applicationState)
|
||||
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException,
|
||||
IllegalArgumentException {
|
||||
if (subSequenceNumber < 0) {
|
||||
throw new IllegalArgumentException("Could not checkpoint at invalid, negative subsequence number "
|
||||
+ subSequenceNumber);
|
||||
throw new IllegalArgumentException(
|
||||
"Could not checkpoint at invalid, negative subsequence number " + subSequenceNumber);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -223,8 +238,11 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
|
|||
&& pendingCheckpoint.compareTo(largestPermittedCheckpointValue) <= 0) {
|
||||
|
||||
if (log.isDebugEnabled()) {
|
||||
log.debug("Preparing checkpoint {}, token {} at specific extended sequence number {}",
|
||||
ShardInfo.getLeaseKey(shardInfo), shardInfo.concurrencyToken(), pendingCheckpoint);
|
||||
log.debug(
|
||||
"Preparing checkpoint {}, token {} at specific extended sequence number {}",
|
||||
ShardInfo.getLeaseKey(shardInfo),
|
||||
shardInfo.concurrencyToken(),
|
||||
pendingCheckpoint);
|
||||
}
|
||||
return doPrepareCheckpoint(pendingCheckpoint, applicationState);
|
||||
} else {
|
||||
|
|
@ -258,7 +276,6 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
|
|||
this.sequenceNumberAtShardEnd = extendedSequenceNumber;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Internal API - has package level access only for testing purposes.
|
||||
*
|
||||
|
|
@ -287,12 +304,18 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
|
|||
if (extendedSequenceNumber != null && !extendedSequenceNumber.equals(lastCheckpointValue)) {
|
||||
try {
|
||||
if (log.isDebugEnabled()) {
|
||||
log.debug("Setting {}, token {} checkpoint to {}", ShardInfo.getLeaseKey(shardInfo),
|
||||
shardInfo.concurrencyToken(), checkpointToRecord);
|
||||
log.debug(
|
||||
"Setting {}, token {} checkpoint to {}",
|
||||
ShardInfo.getLeaseKey(shardInfo),
|
||||
shardInfo.concurrencyToken(),
|
||||
checkpointToRecord);
|
||||
}
|
||||
checkpointer.setCheckpoint(ShardInfo.getLeaseKey(shardInfo), checkpointToRecord, shardInfo.concurrencyToken());
|
||||
checkpointer.setCheckpoint(
|
||||
ShardInfo.getLeaseKey(shardInfo), checkpointToRecord, shardInfo.concurrencyToken());
|
||||
lastCheckpointValue = checkpointToRecord;
|
||||
} catch (ThrottlingException | ShutdownException | InvalidStateException
|
||||
} catch (ThrottlingException
|
||||
| ShutdownException
|
||||
| InvalidStateException
|
||||
| KinesisClientLibDependencyException e) {
|
||||
throw e;
|
||||
} catch (KinesisClientLibException e) {
|
||||
|
|
@ -325,7 +348,8 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
|
|||
* @throws ThrottlingException
|
||||
* @throws ShutdownException
|
||||
*/
|
||||
private PreparedCheckpointer doPrepareCheckpoint(ExtendedSequenceNumber extendedSequenceNumber, byte[] applicationState)
|
||||
private PreparedCheckpointer doPrepareCheckpoint(
|
||||
ExtendedSequenceNumber extendedSequenceNumber, byte[] applicationState)
|
||||
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
|
||||
|
||||
ExtendedSequenceNumber newPrepareCheckpoint = extendedSequenceNumber;
|
||||
|
|
@ -343,8 +367,14 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
|
|||
}
|
||||
|
||||
try {
|
||||
checkpointer.prepareCheckpoint(ShardInfo.getLeaseKey(shardInfo), newPrepareCheckpoint, shardInfo.concurrencyToken(), applicationState);
|
||||
} catch (ThrottlingException | ShutdownException | InvalidStateException
|
||||
checkpointer.prepareCheckpoint(
|
||||
ShardInfo.getLeaseKey(shardInfo),
|
||||
newPrepareCheckpoint,
|
||||
shardInfo.concurrencyToken(),
|
||||
applicationState);
|
||||
} catch (ThrottlingException
|
||||
| ShutdownException
|
||||
| InvalidStateException
|
||||
| KinesisClientLibDependencyException e) {
|
||||
throw e;
|
||||
} catch (KinesisClientLibException e) {
|
||||
|
|
|
|||
|
|
@ -29,9 +29,8 @@ import software.amazon.kinesis.processor.Checkpointer;
|
|||
@KinesisClientInternalApi
|
||||
public class DynamoDBCheckpointFactory implements CheckpointFactory {
|
||||
@Override
|
||||
public Checkpointer createCheckpointer(final LeaseCoordinator leaseLeaseCoordinator,
|
||||
final LeaseRefresher leaseRefresher) {
|
||||
public Checkpointer createCheckpointer(
|
||||
final LeaseCoordinator leaseLeaseCoordinator, final LeaseRefresher leaseRefresher) {
|
||||
return new DynamoDBCheckpointer(leaseLeaseCoordinator, leaseRefresher);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ import java.util.Objects;
|
|||
import java.util.UUID;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
|
||||
import lombok.NonNull;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
|
@ -48,14 +47,16 @@ import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber;
|
|||
public class DynamoDBCheckpointer implements Checkpointer {
|
||||
@NonNull
|
||||
private final LeaseCoordinator leaseCoordinator;
|
||||
|
||||
@NonNull
|
||||
private final LeaseRefresher leaseRefresher;
|
||||
|
||||
private String operation;
|
||||
|
||||
@Override
|
||||
public void setCheckpoint(final String leaseKey, final ExtendedSequenceNumber checkpointValue,
|
||||
final String concurrencyToken) throws KinesisClientLibException {
|
||||
public void setCheckpoint(
|
||||
final String leaseKey, final ExtendedSequenceNumber checkpointValue, final String concurrencyToken)
|
||||
throws KinesisClientLibException {
|
||||
try {
|
||||
boolean wasSuccessful = setCheckpoint(leaseKey, checkpointValue, UUID.fromString(concurrencyToken));
|
||||
if (!wasSuccessful) {
|
||||
|
|
@ -97,17 +98,22 @@ public class DynamoDBCheckpointer implements Checkpointer {
|
|||
}
|
||||
|
||||
@Override
|
||||
public void prepareCheckpoint(final String leaseKey, final ExtendedSequenceNumber pendingCheckpoint,
|
||||
final String concurrencyToken) throws KinesisClientLibException {
|
||||
public void prepareCheckpoint(
|
||||
final String leaseKey, final ExtendedSequenceNumber pendingCheckpoint, final String concurrencyToken)
|
||||
throws KinesisClientLibException {
|
||||
prepareCheckpoint(leaseKey, pendingCheckpoint, concurrencyToken, null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void prepareCheckpoint(String leaseKey, ExtendedSequenceNumber pendingCheckpoint, String concurrencyToken,
|
||||
byte[] pendingCheckpointState) throws KinesisClientLibException {
|
||||
public void prepareCheckpoint(
|
||||
String leaseKey,
|
||||
ExtendedSequenceNumber pendingCheckpoint,
|
||||
String concurrencyToken,
|
||||
byte[] pendingCheckpointState)
|
||||
throws KinesisClientLibException {
|
||||
try {
|
||||
boolean wasSuccessful =
|
||||
prepareCheckpoint(leaseKey, pendingCheckpoint, UUID.fromString(concurrencyToken), pendingCheckpointState);
|
||||
boolean wasSuccessful = prepareCheckpoint(
|
||||
leaseKey, pendingCheckpoint, UUID.fromString(concurrencyToken), pendingCheckpointState);
|
||||
if (!wasSuccessful) {
|
||||
throw new ShutdownException(
|
||||
"Can't prepare checkpoint - instance doesn't hold the lease for this shard");
|
||||
|
|
@ -128,8 +134,10 @@ public class DynamoDBCheckpointer implements Checkpointer {
|
|||
throws DependencyException, InvalidStateException, ProvisionedThroughputException {
|
||||
Lease lease = leaseCoordinator.getCurrentlyHeldLease(leaseKey);
|
||||
if (lease == null) {
|
||||
log.info("Worker {} could not update checkpoint for shard {} because it does not hold the lease",
|
||||
leaseCoordinator.workerIdentifier(), leaseKey);
|
||||
log.info(
|
||||
"Worker {} could not update checkpoint for shard {} because it does not hold the lease",
|
||||
leaseCoordinator.workerIdentifier(),
|
||||
leaseKey);
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
@ -141,12 +149,18 @@ public class DynamoDBCheckpointer implements Checkpointer {
|
|||
return leaseCoordinator.updateLease(lease, concurrencyToken, operation, leaseKey);
|
||||
}
|
||||
|
||||
boolean prepareCheckpoint(String leaseKey, ExtendedSequenceNumber pendingCheckpoint, UUID concurrencyToken, byte[] pendingCheckpointState)
|
||||
boolean prepareCheckpoint(
|
||||
String leaseKey,
|
||||
ExtendedSequenceNumber pendingCheckpoint,
|
||||
UUID concurrencyToken,
|
||||
byte[] pendingCheckpointState)
|
||||
throws DependencyException, InvalidStateException, ProvisionedThroughputException {
|
||||
Lease lease = leaseCoordinator.getCurrentlyHeldLease(leaseKey);
|
||||
if (lease == null) {
|
||||
log.info("Worker {} could not prepare checkpoint for shard {} because it does not hold the lease",
|
||||
leaseCoordinator.workerIdentifier(), leaseKey);
|
||||
log.info(
|
||||
"Worker {} could not prepare checkpoint for shard {} because it does not hold the lease",
|
||||
leaseCoordinator.workerIdentifier(),
|
||||
leaseKey);
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,32 @@
|
|||
package software.amazon.kinesis.common;
|
||||
|
||||
import lombok.NonNull;
|
||||
import software.amazon.awssdk.arns.Arn;
|
||||
import software.amazon.awssdk.regions.Region;
|
||||
import software.amazon.kinesis.annotations.KinesisClientInternalApi;
|
||||
|
||||
import static software.amazon.awssdk.services.kinesis.KinesisAsyncClient.SERVICE_NAME;
|
||||
|
||||
@KinesisClientInternalApi
|
||||
public final class ArnUtil {
|
||||
private static final String STREAM_RESOURCE_PREFIX = "stream/";
|
||||
|
||||
/**
|
||||
* Construct a Kinesis stream ARN.
|
||||
*
|
||||
* @param region The region the stream exists in.
|
||||
* @param accountId The account the stream belongs to.
|
||||
* @param streamName The name of the stream.
|
||||
* @return The {@link Arn} of the Kinesis stream.
|
||||
*/
|
||||
public static Arn constructStreamArn(
|
||||
@NonNull final Region region, @NonNull final String accountId, @NonNull final String streamName) {
|
||||
return Arn.builder()
|
||||
.partition(region.metadata().partition().id())
|
||||
.service(SERVICE_NAME)
|
||||
.region(region.id())
|
||||
.accountId(accountId)
|
||||
.resource(STREAM_RESOURCE_PREFIX + streamName)
|
||||
.build();
|
||||
}
|
||||
}
|
||||
|
|
@ -15,10 +15,8 @@
|
|||
|
||||
package software.amazon.kinesis.common;
|
||||
|
||||
|
||||
public class CommonCalculations {
|
||||
|
||||
|
||||
/**
|
||||
* Convenience method for calculating renewer intervals in milliseconds.
|
||||
*
|
||||
|
|
|
|||
|
|
@ -19,12 +19,11 @@ import java.util.function.Function;
|
|||
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.Getter;
|
||||
import lombok.NonNull;
|
||||
import lombok.Setter;
|
||||
import lombok.ToString;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import lombok.NonNull;
|
||||
import lombok.experimental.Accessors;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import software.amazon.awssdk.arns.Arn;
|
||||
import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient;
|
||||
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
|
||||
|
|
@ -35,9 +34,9 @@ import software.amazon.kinesis.coordinator.CoordinatorConfig;
|
|||
import software.amazon.kinesis.leases.LeaseManagementConfig;
|
||||
import software.amazon.kinesis.lifecycle.LifecycleConfig;
|
||||
import software.amazon.kinesis.metrics.MetricsConfig;
|
||||
import software.amazon.kinesis.processor.MultiStreamTracker;
|
||||
import software.amazon.kinesis.processor.ProcessorConfig;
|
||||
import software.amazon.kinesis.processor.ShardRecordProcessorFactory;
|
||||
import software.amazon.kinesis.processor.MultiStreamTracker;
|
||||
import software.amazon.kinesis.processor.SingleStreamTracker;
|
||||
import software.amazon.kinesis.processor.StreamTracker;
|
||||
import software.amazon.kinesis.retrieval.RetrievalConfig;
|
||||
|
|
@ -45,7 +44,10 @@ import software.amazon.kinesis.retrieval.RetrievalConfig;
|
|||
/**
|
||||
* This Builder is useful to create all configurations for the KCL with default values.
|
||||
*/
|
||||
@Getter @Setter @ToString @EqualsAndHashCode
|
||||
@Getter
|
||||
@Setter
|
||||
@ToString
|
||||
@EqualsAndHashCode
|
||||
@Accessors(fluent = true)
|
||||
public class ConfigsBuilder {
|
||||
/**
|
||||
|
|
@ -139,11 +141,16 @@ public class ConfigsBuilder {
|
|||
* @param workerIdentifier
|
||||
* @param shardRecordProcessorFactory
|
||||
*/
|
||||
public ConfigsBuilder(@NonNull String streamName, @NonNull String applicationName,
|
||||
@NonNull KinesisAsyncClient kinesisClient, @NonNull DynamoDbAsyncClient dynamoDBClient,
|
||||
@NonNull CloudWatchAsyncClient cloudWatchClient, @NonNull String workerIdentifier,
|
||||
public ConfigsBuilder(
|
||||
@NonNull String streamName,
|
||||
@NonNull String applicationName,
|
||||
@NonNull KinesisAsyncClient kinesisClient,
|
||||
@NonNull DynamoDbAsyncClient dynamoDBClient,
|
||||
@NonNull CloudWatchAsyncClient cloudWatchClient,
|
||||
@NonNull String workerIdentifier,
|
||||
@NonNull ShardRecordProcessorFactory shardRecordProcessorFactory) {
|
||||
this(new SingleStreamTracker(streamName),
|
||||
this(
|
||||
new SingleStreamTracker(streamName),
|
||||
applicationName,
|
||||
kinesisClient,
|
||||
dynamoDBClient,
|
||||
|
|
@ -163,11 +170,16 @@ public class ConfigsBuilder {
|
|||
* @param workerIdentifier
|
||||
* @param shardRecordProcessorFactory
|
||||
*/
|
||||
public ConfigsBuilder(@NonNull Arn streamArn, @NonNull String applicationName,
|
||||
@NonNull KinesisAsyncClient kinesisClient, @NonNull DynamoDbAsyncClient dynamoDBClient,
|
||||
@NonNull CloudWatchAsyncClient cloudWatchClient, @NonNull String workerIdentifier,
|
||||
public ConfigsBuilder(
|
||||
@NonNull Arn streamArn,
|
||||
@NonNull String applicationName,
|
||||
@NonNull KinesisAsyncClient kinesisClient,
|
||||
@NonNull DynamoDbAsyncClient dynamoDBClient,
|
||||
@NonNull CloudWatchAsyncClient cloudWatchClient,
|
||||
@NonNull String workerIdentifier,
|
||||
@NonNull ShardRecordProcessorFactory shardRecordProcessorFactory) {
|
||||
this(new SingleStreamTracker(streamArn),
|
||||
this(
|
||||
new SingleStreamTracker(streamArn),
|
||||
applicationName,
|
||||
kinesisClient,
|
||||
dynamoDBClient,
|
||||
|
|
@ -187,9 +199,13 @@ public class ConfigsBuilder {
|
|||
* @param workerIdentifier
|
||||
* @param shardRecordProcessorFactory
|
||||
*/
|
||||
public ConfigsBuilder(@NonNull StreamTracker streamTracker, @NonNull String applicationName,
|
||||
@NonNull KinesisAsyncClient kinesisClient, @NonNull DynamoDbAsyncClient dynamoDBClient,
|
||||
@NonNull CloudWatchAsyncClient cloudWatchClient, @NonNull String workerIdentifier,
|
||||
public ConfigsBuilder(
|
||||
@NonNull StreamTracker streamTracker,
|
||||
@NonNull String applicationName,
|
||||
@NonNull KinesisAsyncClient kinesisClient,
|
||||
@NonNull DynamoDbAsyncClient dynamoDBClient,
|
||||
@NonNull CloudWatchAsyncClient cloudWatchClient,
|
||||
@NonNull String workerIdentifier,
|
||||
@NonNull ShardRecordProcessorFactory shardRecordProcessorFactory) {
|
||||
this.applicationName = applicationName;
|
||||
this.kinesisClient = kinesisClient;
|
||||
|
|
@ -209,8 +225,11 @@ public class ConfigsBuilder {
|
|||
|
||||
public void streamTracker(StreamTracker streamTracker) {
|
||||
this.streamTracker = streamTracker;
|
||||
this.appStreamTracker = DeprecationUtils.convert(streamTracker,
|
||||
singleStreamTracker -> singleStreamTracker.streamConfigList().get(0).streamIdentifier().streamName());
|
||||
this.appStreamTracker = DeprecationUtils.convert(streamTracker, singleStreamTracker -> singleStreamTracker
|
||||
.streamConfigList()
|
||||
.get(0)
|
||||
.streamIdentifier()
|
||||
.streamName());
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -237,7 +256,8 @@ public class ConfigsBuilder {
|
|||
* @return LeaseManagementConfig
|
||||
*/
|
||||
public LeaseManagementConfig leaseManagementConfig() {
|
||||
return new LeaseManagementConfig(tableName(), dynamoDBClient(), kinesisClient(), workerIdentifier());
|
||||
return new LeaseManagementConfig(
|
||||
tableName(), applicationName(), dynamoDBClient(), kinesisClient(), workerIdentifier());
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -0,0 +1,76 @@
|
|||
/*
|
||||
* Copyright 2024 Amazon.com, Inc. or its affiliates.
|
||||
* Licensed under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package software.amazon.kinesis.common;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.experimental.Accessors;
|
||||
import software.amazon.awssdk.services.dynamodb.model.BillingMode;
|
||||
import software.amazon.awssdk.services.dynamodb.model.Tag;
|
||||
|
||||
/**
|
||||
* Configurations of a DDB table created by KCL for its internal operations.
|
||||
*/
|
||||
@Data
|
||||
@Accessors(fluent = true)
|
||||
@NoArgsConstructor
|
||||
public class DdbTableConfig {
|
||||
|
||||
protected DdbTableConfig(final String applicationName, final String tableSuffix) {
|
||||
this.tableName = applicationName + "-" + tableSuffix;
|
||||
}
|
||||
|
||||
/**
|
||||
* name to use for the DDB table. If null, it will default to
|
||||
* applicationName-tableSuffix. If multiple KCL applications
|
||||
* run in the same account, a unique tableName must be provided.
|
||||
*/
|
||||
private String tableName;
|
||||
|
||||
/**
|
||||
* Billing mode used to create the DDB table.
|
||||
*/
|
||||
private BillingMode billingMode = BillingMode.PAY_PER_REQUEST;
|
||||
|
||||
/**
|
||||
* read capacity to provision during DDB table creation,
|
||||
* if billing mode is PROVISIONED.
|
||||
*/
|
||||
private long readCapacity;
|
||||
|
||||
/**
|
||||
* write capacity to provision during DDB table creation,
|
||||
* if billing mode is PROVISIONED.
|
||||
*/
|
||||
private long writeCapacity;
|
||||
|
||||
/**
|
||||
* Flag to enable Point in Time Recovery on the DDB table.
|
||||
*/
|
||||
private boolean pointInTimeRecoveryEnabled = false;
|
||||
|
||||
/**
|
||||
* Flag to enable deletion protection on the DDB table.
|
||||
*/
|
||||
private boolean deletionProtectionEnabled = false;
|
||||
|
||||
/**
|
||||
* Tags to add to the DDB table.
|
||||
*/
|
||||
private Collection<Tag> tags = Collections.emptyList();
|
||||
}
|
||||
|
|
@ -39,8 +39,7 @@ public final class DeprecationUtils {
|
|||
*/
|
||||
@Deprecated
|
||||
public static <R> Either<MultiStreamTracker, R> convert(
|
||||
StreamTracker streamTracker,
|
||||
Function<SingleStreamTracker, R> converter) {
|
||||
StreamTracker streamTracker, Function<SingleStreamTracker, R> converter) {
|
||||
if (streamTracker instanceof MultiStreamTracker) {
|
||||
return Either.left((MultiStreamTracker) streamTracker);
|
||||
} else if (streamTracker instanceof SingleStreamTracker) {
|
||||
|
|
@ -49,5 +48,4 @@ public final class DeprecationUtils {
|
|||
throw new IllegalArgumentException("Unhandled StreamTracker: " + streamTracker);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,12 +15,12 @@
|
|||
|
||||
package software.amazon.kinesis.common;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import software.amazon.kinesis.annotations.KinesisClientInternalApi;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import software.amazon.kinesis.annotations.KinesisClientInternalApi;
|
||||
|
||||
import static software.amazon.kinesis.lifecycle.ShardConsumer.MAX_TIME_BETWEEN_REQUEST_RESPONSE;
|
||||
|
||||
@KinesisClientInternalApi
|
||||
|
|
@ -32,18 +32,22 @@ public class DiagnosticUtils {
|
|||
* @param enqueueTimestamp of the event submitted to the executor service
|
||||
* @param log Slf4j Logger from RecordPublisher to log the events
|
||||
*/
|
||||
public static void takeDelayedDeliveryActionIfRequired(String resourceIdentifier, Instant enqueueTimestamp, Logger log) {
|
||||
final long durationBetweenEnqueueAndAckInMillis = Duration
|
||||
.between(enqueueTimestamp, Instant.now()).toMillis();
|
||||
public static void takeDelayedDeliveryActionIfRequired(
|
||||
String resourceIdentifier, Instant enqueueTimestamp, Logger log) {
|
||||
final long durationBetweenEnqueueAndAckInMillis =
|
||||
Duration.between(enqueueTimestamp, Instant.now()).toMillis();
|
||||
if (durationBetweenEnqueueAndAckInMillis > MAX_TIME_BETWEEN_REQUEST_RESPONSE / 3) {
|
||||
// The above condition logs the warn msg if the delivery time exceeds 11 seconds.
|
||||
log.warn(
|
||||
"{}: Record delivery time to shard consumer is high at {} millis. Check the ExecutorStateEvent logs"
|
||||
+ " to see the state of the executor service. Also check if the RecordProcessor's processing "
|
||||
+ "time is high. ",
|
||||
resourceIdentifier, durationBetweenEnqueueAndAckInMillis);
|
||||
resourceIdentifier,
|
||||
durationBetweenEnqueueAndAckInMillis);
|
||||
} else if (log.isDebugEnabled()) {
|
||||
log.debug("{}: Record delivery time to shard consumer is {} millis", resourceIdentifier,
|
||||
log.debug(
|
||||
"{}: Record delivery time to shard consumer is {} millis",
|
||||
resourceIdentifier,
|
||||
durationBetweenEnqueueAndAckInMillis);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,10 +15,13 @@
|
|||
package software.amazon.kinesis.common;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.CompletionException;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
public class FutureUtils {
|
||||
|
||||
|
|
@ -32,4 +35,14 @@ public class FutureUtils {
|
|||
}
|
||||
}
|
||||
|
||||
public static <T> T unwrappingFuture(final Supplier<CompletableFuture<T>> supplier) {
|
||||
try {
|
||||
return supplier.get().join();
|
||||
} catch (CompletionException e) {
|
||||
if (e.getCause() instanceof RuntimeException) {
|
||||
throw (RuntimeException) e.getCause();
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,14 +15,14 @@
|
|||
|
||||
package software.amazon.kinesis.common;
|
||||
|
||||
import java.math.BigInteger;
|
||||
|
||||
import lombok.NonNull;
|
||||
import lombok.Value;
|
||||
import lombok.experimental.Accessors;
|
||||
import org.apache.commons.lang3.Validate;
|
||||
import software.amazon.awssdk.services.kinesis.model.HashKeyRange;
|
||||
|
||||
import java.math.BigInteger;
|
||||
|
||||
/**
|
||||
* Lease POJO to hold the starting hashkey range and ending hashkey range of kinesis shards.
|
||||
*/
|
||||
|
|
@ -34,8 +34,11 @@ public class HashKeyRangeForLease {
|
|||
private final BigInteger endingHashKey;
|
||||
|
||||
public HashKeyRangeForLease(BigInteger startingHashKey, BigInteger endingHashKey) {
|
||||
Validate.isTrue(startingHashKey.compareTo(endingHashKey) < 0,
|
||||
"StartingHashKey %s must be less than EndingHashKey %s ", startingHashKey, endingHashKey);
|
||||
Validate.isTrue(
|
||||
startingHashKey.compareTo(endingHashKey) < 0,
|
||||
"StartingHashKey %s must be less than EndingHashKey %s ",
|
||||
startingHashKey,
|
||||
endingHashKey);
|
||||
this.startingHashKey = startingHashKey;
|
||||
this.endingHashKey = endingHashKey;
|
||||
}
|
||||
|
|
@ -65,11 +68,15 @@ public class HashKeyRangeForLease {
|
|||
* @param endingHashKeyStr
|
||||
* @return HashKeyRangeForLease
|
||||
*/
|
||||
public static HashKeyRangeForLease deserialize(@NonNull String startingHashKeyStr, @NonNull String endingHashKeyStr) {
|
||||
public static HashKeyRangeForLease deserialize(
|
||||
@NonNull String startingHashKeyStr, @NonNull String endingHashKeyStr) {
|
||||
final BigInteger startingHashKey = new BigInteger(startingHashKeyStr);
|
||||
final BigInteger endingHashKey = new BigInteger(endingHashKeyStr);
|
||||
Validate.isTrue(startingHashKey.compareTo(endingHashKey) < 0,
|
||||
"StartingHashKey %s must be less than EndingHashKey %s ", startingHashKeyStr, endingHashKeyStr);
|
||||
Validate.isTrue(
|
||||
startingHashKey.compareTo(endingHashKey) < 0,
|
||||
"StartingHashKey %s must be less than EndingHashKey %s ",
|
||||
startingHashKeyStr,
|
||||
endingHashKeyStr);
|
||||
return new HashKeyRangeForLease(startingHashKey, endingHashKey);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -14,16 +14,17 @@
|
|||
*/
|
||||
package software.amazon.kinesis.common;
|
||||
|
||||
import java.util.Date;
|
||||
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.ToString;
|
||||
|
||||
import java.util.Date;
|
||||
|
||||
/**
|
||||
* Class that houses the entities needed to specify the position in the stream from where a new application should
|
||||
* start.
|
||||
*/
|
||||
@ToString @EqualsAndHashCode
|
||||
@ToString
|
||||
@EqualsAndHashCode
|
||||
public class InitialPositionInStreamExtended {
|
||||
|
||||
private final InitialPositionInStream position;
|
||||
|
|
|
|||
|
|
@ -15,14 +15,14 @@
|
|||
|
||||
package software.amazon.kinesis.common;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
import software.amazon.awssdk.http.Protocol;
|
||||
import software.amazon.awssdk.http.nio.netty.Http2Configuration;
|
||||
import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient;
|
||||
import software.amazon.awssdk.services.kinesis.KinesisAsyncClient;
|
||||
import software.amazon.awssdk.services.kinesis.KinesisAsyncClientBuilder;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
/**
|
||||
* Utility to setup KinesisAsyncClient to be used with KCL.
|
||||
*/
|
||||
|
|
@ -42,9 +42,12 @@ public class KinesisClientUtil {
|
|||
}
|
||||
|
||||
public static KinesisAsyncClientBuilder adjustKinesisClientBuilder(KinesisAsyncClientBuilder builder) {
|
||||
return builder.httpClientBuilder(NettyNioAsyncHttpClient.builder().maxConcurrency(Integer.MAX_VALUE)
|
||||
.http2Configuration(Http2Configuration.builder().initialWindowSize(INITIAL_WINDOW_SIZE_BYTES)
|
||||
.healthCheckPingPeriod(Duration.ofMillis(HEALTH_CHECK_PING_PERIOD_MILLIS)).build())
|
||||
return builder.httpClientBuilder(NettyNioAsyncHttpClient.builder()
|
||||
.maxConcurrency(Integer.MAX_VALUE)
|
||||
.http2Configuration(Http2Configuration.builder()
|
||||
.initialWindowSize(INITIAL_WINDOW_SIZE_BYTES)
|
||||
.healthCheckPingPeriod(Duration.ofMillis(HEALTH_CHECK_PING_PERIOD_MILLIS))
|
||||
.build())
|
||||
.protocol(Protocol.HTTP2));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -63,12 +63,11 @@ public class KinesisRequestsBuilder {
|
|||
|
||||
@SuppressWarnings("unchecked")
|
||||
private static <T extends AwsRequest.Builder> T appendUserAgent(final T builder) {
|
||||
return (T) builder
|
||||
.overrideConfiguration(
|
||||
AwsRequestOverrideConfiguration.builder()
|
||||
.addApiName(ApiName.builder().name(RetrievalConfig.KINESIS_CLIENT_LIB_USER_AGENT)
|
||||
.version(RetrievalConfig.KINESIS_CLIENT_LIB_USER_AGENT_VERSION).build())
|
||||
return (T) builder.overrideConfiguration(AwsRequestOverrideConfiguration.builder()
|
||||
.addApiName(ApiName.builder()
|
||||
.name(RetrievalConfig.KINESIS_CLIENT_LIB_USER_AGENT)
|
||||
.version(RetrievalConfig.KINESIS_CLIENT_LIB_USER_AGENT_VERSION)
|
||||
.build())
|
||||
.build());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,10 +15,10 @@
|
|||
|
||||
package software.amazon.kinesis.common;
|
||||
|
||||
import lombok.experimental.Accessors;
|
||||
|
||||
import java.util.Optional;
|
||||
|
||||
import lombok.experimental.Accessors;
|
||||
|
||||
@Accessors(fluent = true)
|
||||
public class RequestDetails {
|
||||
|
||||
|
|
@ -62,6 +62,4 @@ public class RequestDetails {
|
|||
public String toString() {
|
||||
return String.format("request id - %s, timestamp - %s", getRequestId(), getTimestamp());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright 2019 Amazon.com, Inc. or its affiliates.
|
||||
* Copyright 2024 Amazon.com, Inc. or its affiliates.
|
||||
* Licensed under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
|
|
@ -12,18 +12,16 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package software.amazon.kinesis.common;
|
||||
|
||||
package software.amazon.kinesis.leases.dynamodb;
|
||||
public class StackTraceUtils {
|
||||
public static String getPrintableStackTrace(final StackTraceElement[] stackTrace) {
|
||||
final StringBuilder stackTraceString = new StringBuilder();
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
/**
|
||||
* This class is just a holder for initial lease table IOPs units. This class will be removed in a future release.
|
||||
*/
|
||||
@Deprecated
|
||||
@NoArgsConstructor(access = AccessLevel.PRIVATE)
|
||||
public class TableConstants {
|
||||
public static final long DEFAULT_INITIAL_LEASE_TABLE_READ_CAPACITY = 10L;
|
||||
public static final long DEFAULT_INITIAL_LEASE_TABLE_WRITE_CAPACITY = 10L;
|
||||
for (final StackTraceElement traceElement : stackTrace) {
|
||||
stackTraceString.append("\tat ").append(traceElement).append("\n");
|
||||
}
|
||||
|
||||
return stackTraceString.toString();
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue