luciano/amazon-kinesis-client
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into 1051-user-max-time

Browse source
This commit is contained in:
fommil 2024-08-21 17:42:52 +01:00 committed by GitHub
commit b0d25757cc
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
362 changed files with 19012 additions and 11754 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.git-blame-ignore-revs Normal file
View file

@ -0,0 +1 @@
63ff312818a5f70eab9ec5bf80b53bdd7bf80248

6
.github/dependabot.yml vendored
View file

@ -2,12 +2,18 @@ version: 2
updates:
- package-ecosystem: "maven"
directory: "/"
labels:
- "dependencies"
- "v2.x"
schedule:
interval: "weekly"
# branch - v1.x
- package-ecosystem: "maven"
directory: "/"
labels:
- "dependencies"
- "v1.x"
target-branch: "v1.x"
schedule:
interval: "weekly"

128
.github/scripts/backwards_compatibility_check.sh vendored Executable file
View file

@ -0,0 +1,128 @@
#!/bin/bash
TRUE=1
FALSE=0
KCL_MAVEN_DIR=~/.m2/repository/software/amazon/kinesis/amazon-kinesis-client
REMOVED_METHODS_FLAG=$FALSE
LATEST_VERSION=""
LATEST_JAR=""
CURRENT_VERSION=""
CURRENT_JAR=""
# Get the JAR from the latest version release on Maven.
get_latest_jar() {
# clear the directory so that the latest release will be the only version in the Maven directory after running mvn dependency:get
rm -rf "$KCL_MAVEN_DIR"
mvn -B dependency:get -Dartifact=software.amazon.kinesis:amazon-kinesis-client:LATEST
LATEST_VERSION=$(ls "$KCL_MAVEN_DIR" | grep -E '[0-9]+.[0-9]+.[0-9]+')
LATEST_JAR=$KCL_MAVEN_DIR/$LATEST_VERSION/amazon-kinesis-client-$LATEST_VERSION.jar
}
# Get the JAR with the changes that need to be verified.
get_current_jar() {
mvn -B install -Dmaven.test.skip=true
CURRENT_VERSION=$(mvn -q -Dexec.executable=echo -Dexec.args='${project.version}' --non-recursive exec:exec)
CURRENT_JAR=$KCL_MAVEN_DIR/$CURRENT_VERSION/amazon-kinesis-client-$CURRENT_VERSION.jar
}
is_new_minor_release() {
local latest_minor_version=$(echo "$LATEST_VERSION" | cut -d . -f 2)
local current_minor_version=$(echo "$CURRENT_VERSION" | cut -d . -f 2)
[[ "$latest_minor_version" != "$current_minor_version" ]]
return $?
}
# Skip classes with the KinesisClientInternalApi annotation. These classes are subject to breaking backwards compatibility.
is_kinesis_client_internal_api() {
local current_class="$1"
local grep_internal_api_result=$(javap -v -classpath "$LATEST_JAR" "$current_class" | grep KinesisClientInternalApi)
[[ "$grep_internal_api_result" != "" ]]
return $?
}
# Skip classes which are not public (e.g. package level). These classes will not break backwards compatibility.
is_non_public_class() {
local current_class="$1"
local class_definition=$(javap -classpath "$LATEST_JAR" "$current_class" | head -2 | tail -1)
[[ "$class_definition" != *"public"* ]]
return $?
}
# Ignore methods that change from abstract to non-abstract (and vice versa) if the class is an interface.
ignore_abstract_changes_in_interfaces() {
local current_class="$1"
local class_definition=$(javap -classpath "$LATEST_JAR" "$current_class" | head -2 | tail -1)
if [[ $class_definition == *"interface"* ]]
then
LATEST_METHODS=${LATEST_METHODS// abstract / }
CURRENT_METHODS=${CURRENT_METHODS// abstract / }
fi
}
# Checks if there are any methods in the latest version that were removed in the current version.
find_removed_methods() {
echo "Checking if methods in current version (v$CURRENT_VERSION) were removed from latest version (v$LATEST_VERSION)"
if is_new_minor_release
then
echo "New minor release is being performed. Ignoring changes in classes marked with @KinesisClientInternalApi annotation."
fi
local latest_classes=$(
jar tf $LATEST_JAR |
grep .class |
tr / . |
sed 's/\.class$//' |
# skip generated proto classes since these have a lot of inherited methods
# that are not outputted by javap. besides, generated java code is not a
# good indicator of proto compatibility- it will not capture reserved
# tags or deprecated fields.
grep -v 'software\.amazon\.kinesis\.retrieval\.kpl\.Messages')
for class in $latest_classes
do
if (is_kinesis_client_internal_api "$class" && is_new_minor_release) || is_non_public_class "$class"
then
continue
fi
LATEST_METHODS=$(javap -classpath "$LATEST_JAR" "$class")
CURRENT_METHODS=$(javap -classpath "$CURRENT_JAR" "$class")
ignore_abstract_changes_in_interfaces "$class"
local removed_methods=$(diff <(echo "$LATEST_METHODS") <(echo "$CURRENT_METHODS") | grep '^<')
# ignore synthetic access methods - these are not available to users and will not break backwards compatibility
removed_methods=$(echo "$removed_methods" | grep -v "access\$[0-9]\+")
if [[ "$removed_methods" != "" ]]
then
REMOVED_METHODS_FLAG=$TRUE
if is_kinesis_client_internal_api "$class"
then
echo "Found removed methods in class with @KinesisClientInternalApi annotation. To resolve these issues, upgrade the current minor version or address these changes."
fi
echo "$class does not have method(s):"
echo "$removed_methods"
fi
done
}
get_backwards_compatible_result() {
if [[ $REMOVED_METHODS_FLAG == $TRUE ]]
then
echo "Current KCL version $CURRENT_VERSION is not backwards compatible with version $LATEST_VERSION. See output above for removed packages/methods."
exit 1
else
echo "Current KCL version $CURRENT_VERSION is backwards compatible with version $LATEST_VERSION."
exit 0
fi
}
main() {
get_latest_jar
get_current_jar
find_removed_methods
get_backwards_compatible_result
}
main

46
.github/workflows/maven.yml vendored Normal file
View file

@ -0,0 +1,46 @@
# This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-maven
# This workflow uses actions that are not certified by GitHub.
# They are provided by a third-party and are governed by
# separate terms of service, privacy policy, and support
# documentation.
name: Java CI with Maven
on:
push:
branches:
- "master"
pull_request:
branches:
- "master"
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up JDK 8
uses: actions/setup-java@v4
with:
java-version: '8'
distribution: 'corretto'
- name: Build with Maven
run: mvn -B package --file pom.xml -DskipITs
backwards-compatible-check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up JDK 8
uses: actions/setup-java@v4
with:
java-version: '8'
distribution: 'corretto'
- name: Check backwards compatibility of changes
run: .github/scripts/backwards_compatibility_check.sh

3
.gitignore vendored
View file

@ -2,4 +2,5 @@ target/
AwsCredentials.properties
.idea
*.iml
.DS_Store
*.swp
.DS_Store

BIN
.log.swp
View file

Binary file not shown.

130
CHANGELOG.md
View file

@ -3,6 +3,136 @@
For **1.x** release notes, please see [v1.x/CHANGELOG.md](https://github.com/awslabs/amazon-kinesis-client/blob/v1.x/CHANGELOG.md)
---
### Release 2.6.0 (2024-05-01)
* [#1317](https://github.com/awslabs/amazon-kinesis-client/pull/1317) Add enablePriorityLeaseAssignment config
* [#1320](https://github.com/awslabs/amazon-kinesis-client/pull/1320) Update lease taker to get unassigned leases
* [#1318](https://github.com/awslabs/amazon-kinesis-client/pull/1318) Internally construct and use stream ARNs for all streams in multi-stream mode
* [#1291](https://github.com/awslabs/amazon-kinesis-client/pull/1291) Update RetrievalFactory implementations to utilize the StreamIdentifier field of StreamConfig
* [#1308](https://github.com/awslabs/amazon-kinesis-client/pull/1308) Move shutdownComplete call to ShardConsumer
* [#1313](https://github.com/awslabs/amazon-kinesis-client/pull/1313) Add additional integration tests for multi-stream and cross account access
* [#1273](https://github.com/awslabs/amazon-kinesis-client/pull/1273) Optimize currentStreamConfigMap by cleaning up lingering streams
* [#1302](https://github.com/awslabs/amazon-kinesis-client/pull/1302) Fix gracefulShutdown behavior in Scheduler
### Release 2.5.8 (2024-03-27)
* [#1278](https://github.com/awslabs/amazon-kinesis-client/pull/1278) Upgrade awssdk.version from 2.25.3 to 2.25.11
* [#1279](https://github.com/awslabs/amazon-kinesis-client/pull/1279) Upgrade org.apache.maven.plugins:maven-gpg-plugin from 3.1.0 to 3.2.1
* [#1280](https://github.com/awslabs/amazon-kinesis-client/pull/1280) Upgrade org.apache.commons:commons-lang3 from 3.12.0 to 3.14.0
* [#1282](https://github.com/awslabs/amazon-kinesis-client/pull/1282) Upgrade org.apache.maven.plugins:maven-javadoc-plugin from 3.5.0 to 3.6.3
* [#1277](https://github.com/awslabs/amazon-kinesis-client/pull/1277) Reuse 'ShardSyncTaskManager' instance for existing stream to avoid duplicate enqueue of 'ShardSyncTask'
### Release 2.5.7 (2024-03-19)
* [#1275](https://github.com/awslabs/amazon-kinesis-client/pull/1275) Update PollingConfig maxRecords method to return PollingConfig
* [#1236](https://github.com/awslabs/amazon-kinesis-client/pull/1236) Upgrade commons-io:commons-io from 2.11.0 to 2.15.1
* [#1189](https://github.com/awslabs/amazon-kinesis-client/pull/1189) Upgrade org.apache.maven.plugins:maven-resources-plugin from 3.3.0 to 3.3.1
* [#1139](https://github.com/awslabs/amazon-kinesis-client/pull/1139) Upgrade maven-surefire-plugin from 2.22.2 to 3.1.2
* [#1138](https://github.com/awslabs/amazon-kinesis-client/pull/1138) Upgrade maven-failsafe-plugin from 2.22.2 to 3.1.2
* [#1125](https://github.com/awslabs/amazon-kinesis-client/pull/1125) Upgrade maven-gpg-plugin from 3.0.1 to 3.1.0
### Release 2.5.6 (2024-03-08)
* [#1271](https://github.com/awslabs/amazon-kinesis-client/pull/1271) Adding snapshot for 2.5.6-SNAPSHOT
* [#1268](https://github.com/awslabs/amazon-kinesis-client/pull/1268) Upgrade ch.qos.logback:logback-classic dependency from 1.3.12 to 1.3.14
* [#1268](https://github.com/awslabs/amazon-kinesis-client/pull/1268) Upgrade awssdk.version from 2.20.43 to 2.25.3
* [#1268](https://github.com/awslabs/amazon-kinesis-client/pull/1268) Upgrade aws-java-sdk.version from 1.12.405 to 1.12.668
* [#1268](https://github.com/awslabs/amazon-kinesis-client/pull/1268) Upgrade gsr.version from 1.1.17 to 1.1.19
### Release 2.5.5 (2024-02-22)
* [#1257](https://github.com/awslabs/amazon-kinesis-client/pull/1257) Prevent improper error logging during worker shutdown
* [#1260](https://github.com/awslabs/amazon-kinesis-client/pull/1260) Add Deletion protection config
* [#1258](https://github.com/awslabs/amazon-kinesis-client/pull/1258) Fix issue in configuring metricsEnabledDimensions
* [#1259](https://github.com/awslabs/amazon-kinesis-client/pull/1259) Add snapshot to version
### Release 2.5.4 (December 12, 2023)
* [#1232](https://github.com/awslabs/amazon-kinesis-client/pull/1232) Upgrade ch.qos.logback:logback-classic dependency from 1.3.0 to 1.3.12 in /amazon-kinesis-client
* [#1233](https://github.com/awslabs/amazon-kinesis-client/pull/1233) Upgrade ch.qos.logback:logback-classic dependency from 1.3.0 to 1.3.12 in /amazon-kinesis-client-multilang
* [#1230](https://github.com/awslabs/amazon-kinesis-client/pull/1230) Bug fix which now allows MultiLangDaemon to configure idleTimeBetweenReadsInMillis
* [#1229](https://github.com/awslabs/amazon-kinesis-client/pull/1229) Added link to `javadoc.io`-hosted Javadoc in the README
* [#1218](https://github.com/awslabs/amazon-kinesis-client/pull/1218) Added doc for leases and the lease lifecycle to help explain lease lifecycle logic.
* [#1226](https://github.com/awslabs/amazon-kinesis-client/pull/1226) Upgraded KCL from 2.5.3 to 2.5.4-SNAPSHOT
### Release 2.5.3 (November 8, 2023)
* [#1219](https://github.com/awslabs/amazon-kinesis-client/pull/1219) Provided streamArn in getRecords request
* [#1216](https://github.com/awslabs/amazon-kinesis-client/pull/1216) Updated AWS Glue Schema Registry from version 1.1.14 to 1.1.17.
* [#1205](https://github.com/awslabs/amazon-kinesis-client/pull/1205) Updated the FAQ with impact of changing default checkpoint.
* [#1203](https://github.com/awslabs/amazon-kinesis-client/pull/1203) Added links from README.md to FAQ and doc folder.
* [#1202](https://github.com/awslabs/amazon-kinesis-client/pull/1202) Introduced a FAQ for Kinesis Client Library
* [#1200](https://github.com/awslabs/amazon-kinesis-client/pull/1200) Added test case for StreamIdentifier serialization.
### Release 2.5.2 (August 7, 2023)
* [#1184](https://github.com/awslabs/amazon-kinesis-client/pull/1184) [#367] Enhanced multi-lang `AWSCredentialsProvider=...` decoder and c…
* [#1186](https://github.com/awslabs/amazon-kinesis-client/pull/1186) Provided documentation for multilang's new NestedPropertyKey enhancement.
* [#1181](https://github.com/awslabs/amazon-kinesis-client/pull/1181) CVE-2023-2976: Upgrade Google Guava dependency version from `32.0.0-jre` to `32.1.1-jre`
* [#1159](https://github.com/awslabs/amazon-kinesis-client/pull/1159) Bug fix in lease refresher integration test with occasional failures
* [#1157](https://github.com/awslabs/amazon-kinesis-client/pull/1157) Fix NPE on graceful shutdown before DDB `LeaseCoordinator` starts.
* [#1152](https://github.com/awslabs/amazon-kinesis-client/pull/1152) Adding resharding integration tests and changing ITs to not run by default
* [#1162](https://github.com/awslabs/amazon-kinesis-client/pull/1162) Only deleting resource created by ITs
* [#1158](https://github.com/awslabs/amazon-kinesis-client/pull/1158) Checkstyle: tightened `LineLength` restriction from 170 to 150.
* [#1151](https://github.com/awslabs/amazon-kinesis-client/pull/1151) Modified `dependabot.yml` to set the correct `v[1|2].x` label.
* [#1164](https://github.com/awslabs/amazon-kinesis-client/pull/1164) Upgraded KCL Version from 2.5.1 to 2.5.2-SNAPSHOT
### Release 2.5.1 (June 27, 2023)
* [#1143](https://github.com/awslabs/amazon-kinesis-client/pull/1143) Upgrade MultiLangDaemon to support StreamARN
* [#1145](https://github.com/awslabs/amazon-kinesis-client/pull/1145) Introduced GitHub actions to trigger Maven builds during merge/pull requests
* [#1136](https://github.com/awslabs/amazon-kinesis-client/pull/1136) Added testing architecture and KCL 2.x basic polling/streaming tests
* [#1153](https://github.com/awslabs/amazon-kinesis-client/pull/1153) Checkstyle: added `UnusedImports` check.
* [#1150](https://github.com/awslabs/amazon-kinesis-client/pull/1150) Enabled Checkstyle validation of test resources.
* [#1149](https://github.com/awslabs/amazon-kinesis-client/pull/1149) Bound Checkstyle to `validate` goal for automated enforcement.
* [#1148](https://github.com/awslabs/amazon-kinesis-client/pull/1148) Code cleanup to faciliate Checkstyle enforcement.
* [#1142](https://github.com/awslabs/amazon-kinesis-client/pull/1142) Upgrade Google Guava dependency version from 31.1-jre to 32.0.0-jre
* [#1115](https://github.com/awslabs/amazon-kinesis-client/pull/1115) Update KCL version from 2.5.0 to 2.5.1-SNAPSHOT
### Release 2.5.0 (May 19, 2023)
* **[#1109](https://github.com/awslabs/amazon-kinesis-client/pull/1109) Add support for stream ARNs**
* **[#1065](https://github.com/awslabs/amazon-kinesis-client/pull/1065) Allow tags to be added when lease table is created**
* [#1094](https://github.com/awslabs/amazon-kinesis-client/pull/1094) Code cleanup to introduce better testing
* [#1088](https://github.com/awslabs/amazon-kinesis-client/pull/1088) Minimize race in PSSM to optimize shard sync calls
* [#1086](https://github.com/awslabs/amazon-kinesis-client/pull/1086) Add additional SingleStreamTracker constructor with stream position parameter
* [#1084](https://github.com/awslabs/amazon-kinesis-client/pull/1084) More consistent testing behavior with restartAfterRequestTimerExpires
* [#1066](https://github.com/awslabs/amazon-kinesis-client/pull/1066) More consistent testing behavior with HashRangesAreAlwaysComplete
* [#1072](https://github.com/awslabs/amazon-kinesis-client/pull/1072) Upgrade nexus-staging-maven-plugin from 1.6.8 to 1.6.13
* [#1073](https://github.com/awslabs/amazon-kinesis-client/pull/1073) Upgrade slf4j-api from 2.0.6 to 2.0.7
* [#1090](https://github.com/awslabs/amazon-kinesis-client/pull/1090) Upgrade awssdk.version from 2.20.8 to 2.20.43
* [#1071](https://github.com/awslabs/amazon-kinesis-client/pull/1071) Upgrade maven-compiler-plugin from 3.8.1 to 3.11.0
### Release 2.4.8 (March 21, 2023)
* [#1080](https://github.com/awslabs/amazon-kinesis-client/pull/1080) Added metric in `ShutdownTask` for scenario when parent leases are missing.
* [#1077](https://github.com/awslabs/amazon-kinesis-client/pull/1077) Reverted changes to pom property
* [#1069](https://github.com/awslabs/amazon-kinesis-client/pull/1069) Fixed flaky InitializationWaitsWhenLeaseTableIsEmpty test
### Release 2.4.7 (March 17, 2023)
* **NOTE: Due to an issue during the release process, the 2.4.7 published artifacts are incomplete and non-viable. Please use 2.4.8 or later.**
* [#1063](https://github.com/awslabs/amazon-kinesis-client/pull/1063) Allow leader to learn new leases upon re-election to avoid unnecessary shardSyncs
* [#1060](https://github.com/awslabs/amazon-kinesis-client/pull/1060) Add new metric to be emitted on lease creation
* [#1057](https://github.com/awslabs/amazon-kinesis-client/pull/1057) Added more logging in `Scheduler` w.r.t. `StreamConfig`s.
* [#1059](https://github.com/awslabs/amazon-kinesis-client/pull/1059) DRY: simplification of `HierarchicalShardSyncerTest`.
* [#1062](https://github.com/awslabs/amazon-kinesis-client/pull/1062) Fixed retry storm in `PrefetchRecordsPublisher`.
* [#1061](https://github.com/awslabs/amazon-kinesis-client/pull/1061) Fixed NPE in `LeaseCleanupManager`.
* [#1056](https://github.com/awslabs/amazon-kinesis-client/pull/1056) Clean up in-memory state of deleted kinesis stream in MultiStreamMode
* [#1058](https://github.com/awslabs/amazon-kinesis-client/pull/1058) Documentation: added `<pre>` tags so fixed-format diagrams aren't garbled.
* [#1053](https://github.com/awslabs/amazon-kinesis-client/pull/1053) Exposed convenience method of `ExtendedSequenceNumber#isSentinelCheckpoint()`
* [#1043](https://github.com/awslabs/amazon-kinesis-client/pull/1043) Removed a `.swp` file, and updated `.gitignore`.
* [#1047](https://github.com/awslabs/amazon-kinesis-client/pull/1047) Upgrade awssdk.version from 2.19.31 to 2.20.8
* [#1046](https://github.com/awslabs/amazon-kinesis-client/pull/1046) Upgrade maven-javadoc-plugin from 3.3.1 to 3.5.0
* [#1038](https://github.com/awslabs/amazon-kinesis-client/pull/1038) Upgrade gsr.version from 1.1.13 to 1.1.14
* [#1037](https://github.com/awslabs/amazon-kinesis-client/pull/1037) Upgrade aws-java-sdk.version from 1.12.370 to 1.12.405
### Release 2.4.6 (February 21, 2023)
* [#1041](https://github.com/awslabs/amazon-kinesis-client/pull/1041) Minor optimizations (e.g., calculate-once, put instead of get+put)
* [#1035](https://github.com/awslabs/amazon-kinesis-client/pull/1035) Release Note updates to avoid duplication and bitrot (e.g., 1.x release
* [#935](https://github.com/awslabs/amazon-kinesis-client/pull/935) Pass isAtShardEnd correctly to processRecords call
* [#1040](https://github.com/awslabs/amazon-kinesis-client/pull/1040) Increased logging verbosity around lease management
* [#1024](https://github.com/awslabs/amazon-kinesis-client/pull/1024) Added logging w.r.t. StreamConfig handling.
* [#1034](https://github.com/awslabs/amazon-kinesis-client/pull/1034) Optimization: 9~15% improvement in KinesisDataFetcher wall-time
* [#1045](https://github.com/awslabs/amazon-kinesis-client/pull/1045) Fixed duplication of project version in children pom.xml
* [#956](https://github.com/awslabs/amazon-kinesis-client/pull/956) Fixed warning message typos
* [#795](https://github.com/awslabs/amazon-kinesis-client/pull/795) Fixed log message spacing
* [#740](https://github.com/awslabs/amazon-kinesis-client/pull/740) Fixed typo in Comment
* [#1028](https://github.com/awslabs/amazon-kinesis-client/pull/1028) Refactored MultiStreamTracker to provide and enhance OOP for both
* [#1027](https://github.com/awslabs/amazon-kinesis-client/pull/1027) Removed CHECKSTYLE:OFF toggles which can invite/obscure sub-par code.
* [#1032](https://github.com/awslabs/amazon-kinesis-client/pull/1032) Upgrade rxjava from 3.1.5 to 3.1.6
* [#1030](https://github.com/awslabs/amazon-kinesis-client/pull/1030) Upgrade awssdk.version from 2.19.2 to 2.19.31
* [#1029](https://github.com/awslabs/amazon-kinesis-client/pull/1029) Upgrade slf4j-api from 2.0.0 to 2.0.6
* [#1015](https://github.com/awslabs/amazon-kinesis-client/pull/1015) Upgrade protobuf-java from 3.21.5 to 3.21.12
### Release 2.4.5 (January 04, 2023)
* [#1014](https://github.com/awslabs/amazon-kinesis-client/pull/1014) Use AFTER_SEQUENCE_NUMBER iterator type for expired iterator request

30
README.md
View file

@ -5,6 +5,9 @@ The **Amazon Kinesis Client Library for Java** (Amazon KCL) enables Java develop
* [Kinesis Product Page][kinesis]
* [Forum][kinesis-forum]
* [Javadoc][kcl-javadoc]
* [FAQ](docs/FAQ.md)
* [KCL Documentation](docs/) (folder)
* [Issues][kinesis-client-library-issues]
### Recommended Upgrade for All Users of the 1.x Amazon Kinesis Client
@ -32,9 +35,19 @@ Please open an issue if you have any questions.
## Building from Source
After you've downloaded the code from GitHub, you can build it using Maven. To disable GPG signing in the build, use
this command: `mvn clean install -Dgpg.skip=true`. Note: This command runs Integration tests, which in turn creates AWS
resources (which requires manual cleanup). Integration tests require valid AWS credentials need to be discovered at
runtime. To skip running integration tests, add ` -DskipITs` option to the build command.
this command: `mvn clean install -Dgpg.skip=true`.
Note: This command does not run integration tests.
To disable running unit tests in the build, add the property `-Dskip.ut=true`.
## Running Integration Tests
Note that running integration tests creates AWS resources.
Integration tests require valid AWS credentials.
This will look for a default AWS profile specified in your local `.aws/credentials`.
To run all integration tests: `mvn verify -DskipITs=false`.
To run one integration tests, specify the integration test class: `mvn -Dit.test="BasicStreamConsumerIntegrationTest" -DskipITs=false verify`
Optionally, you can provide the name of an IAM user/role to run tests with as a string using this command: `mvn -DskipITs=false -DawsProfile="<PROFILE_NAME>" verify`.
## Integration with the Kinesis Producer Library
For producer-side developers using the **[Kinesis Producer Library (KPL)][kinesis-guide-kpl]**, the KCL integrates without additional effort. When the KCL retrieves an aggregated Amazon Kinesis record consisting of multiple KPL user records, it will automatically invoke the KPL to extract the individual user records before returning them to the user.
@ -50,7 +63,7 @@ The recommended way to use the KCL for Java is to consume it from Maven.
<dependency>
<groupId>software.amazon.kinesis</groupId>
<artifactId>amazon-kinesis-client</artifactId>
<version>2.4.4</version>
<version>2.6.0</version>
</dependency>
```
@ -71,10 +84,11 @@ The recommended way to use the KCL for Java is to consume it from Maven.
| 2.x | [master/CHANGELOG.md](CHANGELOG.md) |
| 1.x | [v1.x/CHANGELOG.md](https://github.com/awslabs/amazon-kinesis-client/blob/v1.x/CHANGELOG.md) |
[kinesis]: http://aws.amazon.com/kinesis
[kinesis-forum]: http://developer.amazonwebservices.com/connect/forum.jspa?forumID=169
[kinesis-client-library-issues]: https://github.com/awslabs/amazon-kinesis-client/issues
[docs-signup]: http://docs.aws.amazon.com/AWSSdkDocsJava/latest/DeveloperGuide/java-dg-setup.html
[kcl-javadoc]: https://javadoc.io/doc/software.amazon.kinesis/amazon-kinesis-client/
[kinesis]: http://aws.amazon.com/kinesis
[kinesis-client-library-issues]: https://github.com/awslabs/amazon-kinesis-client/issues
[kinesis-forum]: http://developer.amazonwebservices.com/connect/forum.jspa?forumID=169
[kinesis-guide]: http://docs.aws.amazon.com/kinesis/latest/dev/introduction.html
[kinesis-guide-begin]: http://docs.aws.amazon.com/kinesis/latest/dev/before-you-begin.html
[kinesis-guide-create]: http://docs.aws.amazon.com/kinesis/latest/dev/step-one-create-stream.html
@ -83,5 +97,5 @@ The recommended way to use the KCL for Java is to consume it from Maven.
[kinesis-guide-kpl]: http://docs.aws.amazon.com//kinesis/latest/dev/developing-producers-with-kpl.html
[kinesis-guide-consumer-deaggregation]: http://docs.aws.amazon.com//kinesis/latest/dev/kinesis-kpl-consumer-deaggregation.html
[kclpy]: https://github.com/awslabs/amazon-kinesis-client-python
[multi-lang-protocol]: https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/package-info.java
[multi-lang-protocol]: /amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/package-info.java
[migration-guide]: https://docs.aws.amazon.com/streams/latest/dev/kcl-migration.html

37
amazon-kinesis-client-multilang/pom.xml
View file

@ -21,14 +21,14 @@
<parent>
<artifactId>amazon-kinesis-client-pom</artifactId>
<groupId>software.amazon.kinesis</groupId>
<version>${revision}</version>
<version>2.6.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>amazon-kinesis-client-multilang</artifactId>
<properties>
<aws-java-sdk.version>1.12.370</aws-java-sdk.version>
<aws-java-sdk.version>1.12.668</aws-java-sdk.version>
</properties>
<dependencies>
@ -78,7 +78,7 @@
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>1.3.0</version>
<version>1.3.14</version>
</dependency>
<dependency>
<groupId>com.beust</groupId>
@ -88,7 +88,7 @@
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.11.0</version>
<version>2.16.1</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
@ -130,10 +130,9 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.1</version>
<version>3.13.0</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
<release>8</release>
<encoding>UTF-8</encoding>
</configuration>
</plugin>
@ -143,7 +142,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>3.3.1</version>
<version>3.7.0</version>
<executions>
<execution>
<id>attach-javadocs</id>
@ -166,7 +165,29 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>com.diffplug.spotless</groupId>
<artifactId>spotless-maven-plugin</artifactId>
<version>2.30.0</version> <!--last version to support java 8-->
<configuration>
<java>
<palantirJavaFormat />
<importOrder>
<order>java,,\#</order>
</importOrder>
</java>
</configuration>
<executions>
<execution>
<goals>
<goal>check</goal>
</goals>
<phase>compile</phase>
</execution>
</executions>
</plugin>
</plugins>
</build>
<profiles>

224
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/coordinator/KinesisClientLibConfiguration.java
View file

@ -18,11 +18,9 @@ import java.util.Date;
import java.util.Optional;
import java.util.Set;
import org.apache.commons.lang3.Validate;
import com.google.common.collect.ImmutableSet;
import lombok.Getter;
import org.apache.commons.lang3.Validate;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.kinesis.checkpoint.ShardRecordProcessorCheckpointer;
import software.amazon.kinesis.common.InitialPositionInStream;
@ -119,14 +117,16 @@ public class KinesisClientLibConfiguration {
/**
* Metrics dimensions that always will be enabled regardless of the config provided by user.
*/
public static final Set<String> METRICS_ALWAYS_ENABLED_DIMENSIONS = ImmutableSet
.of(MetricsUtil.OPERATION_DIMENSION_NAME);
public static final Set<String> METRICS_ALWAYS_ENABLED_DIMENSIONS =
ImmutableSet.of(MetricsUtil.OPERATION_DIMENSION_NAME);
/**
* Allowed dimensions for CloudWatch metrics. By default, worker ID dimension will be disabled.
*/
public static final Set<String> DEFAULT_METRICS_ENABLED_DIMENSIONS = ImmutableSet.<String> builder()
.addAll(METRICS_ALWAYS_ENABLED_DIMENSIONS).add(MetricsUtil.SHARD_ID_DIMENSION_NAME).build();
public static final Set<String> DEFAULT_METRICS_ENABLED_DIMENSIONS = ImmutableSet.<String>builder()
.addAll(METRICS_ALWAYS_ENABLED_DIMENSIONS)
.add(MetricsUtil.SHARD_ID_DIMENSION_NAME)
.build();
/**
* Metrics dimensions that signify all possible dimensions.
@ -217,6 +217,9 @@ public class KinesisClientLibConfiguration {
private AwsCredentialsProvider dynamoDBCredentialsProvider;
private AwsCredentialsProvider cloudWatchCredentialsProvider;
private long failoverTimeMillis;
private boolean enablePriorityLeaseAssignment;
private boolean leaseTableDeletionProtectionEnabled;
private boolean leaseTablePitrEnabled;
private String workerIdentifier;
private long shardSyncIntervalMillis;
private int maxRecords;
@ -284,8 +287,8 @@ public class KinesisClientLibConfiguration {
* @param workerId
* Used to distinguish different workers/processes of a Kinesis application
*/
public KinesisClientLibConfiguration(String applicationName, String streamName,
AwsCredentialsProvider credentialsProvider, String workerId) {
public KinesisClientLibConfiguration(
String applicationName, String streamName, AwsCredentialsProvider credentialsProvider, String workerId) {
this(applicationName, streamName, credentialsProvider, credentialsProvider, credentialsProvider, workerId);
}
@ -307,16 +310,36 @@ public class KinesisClientLibConfiguration {
* @param workerId
* Used to distinguish different workers/processes of a Kinesis application
*/
public KinesisClientLibConfiguration(String applicationName, String streamName,
AwsCredentialsProvider kinesisCredentialsProvider, AwsCredentialsProvider dynamoDBCredentialsProvider,
AwsCredentialsProvider cloudWatchCredentialsProvider, String workerId) {
this(applicationName, streamName, null, null, DEFAULT_INITIAL_POSITION_IN_STREAM, kinesisCredentialsProvider,
dynamoDBCredentialsProvider, cloudWatchCredentialsProvider, DEFAULT_FAILOVER_TIME_MILLIS, workerId,
DEFAULT_MAX_RECORDS, DEFAULT_IDLETIME_BETWEEN_READS_MILLIS,
DEFAULT_DONT_CALL_PROCESS_RECORDS_FOR_EMPTY_RECORD_LIST, DEFAULT_PARENT_SHARD_POLL_INTERVAL_MILLIS,
DEFAULT_SHARD_SYNC_INTERVAL_MILLIS, DEFAULT_CLEANUP_LEASES_UPON_SHARDS_COMPLETION,
DEFAULT_TASK_BACKOFF_TIME_MILLIS, DEFAULT_METRICS_BUFFER_TIME_MILLIS, DEFAULT_METRICS_MAX_QUEUE_SIZE,
DEFAULT_VALIDATE_SEQUENCE_NUMBER_BEFORE_CHECKPOINTING, null, DEFAULT_SHUTDOWN_GRACE_MILLIS,
public KinesisClientLibConfiguration(
String applicationName,
String streamName,
AwsCredentialsProvider kinesisCredentialsProvider,
AwsCredentialsProvider dynamoDBCredentialsProvider,
AwsCredentialsProvider cloudWatchCredentialsProvider,
String workerId) {
this(
applicationName,
streamName,
null,
null,
DEFAULT_INITIAL_POSITION_IN_STREAM,
kinesisCredentialsProvider,
dynamoDBCredentialsProvider,
cloudWatchCredentialsProvider,
DEFAULT_FAILOVER_TIME_MILLIS,
workerId,
DEFAULT_MAX_RECORDS,
DEFAULT_IDLETIME_BETWEEN_READS_MILLIS,
DEFAULT_DONT_CALL_PROCESS_RECORDS_FOR_EMPTY_RECORD_LIST,
DEFAULT_PARENT_SHARD_POLL_INTERVAL_MILLIS,
DEFAULT_SHARD_SYNC_INTERVAL_MILLIS,
DEFAULT_CLEANUP_LEASES_UPON_SHARDS_COMPLETION,
DEFAULT_TASK_BACKOFF_TIME_MILLIS,
DEFAULT_METRICS_BUFFER_TIME_MILLIS,
DEFAULT_METRICS_MAX_QUEUE_SIZE,
DEFAULT_VALIDATE_SEQUENCE_NUMBER_BEFORE_CHECKPOINTING,
null,
DEFAULT_SHUTDOWN_GRACE_MILLIS,
DEFAULT_SCHEDULER_INITIALIZATION_BACKOFF_TIME_MILLIS);
}
@ -376,20 +399,53 @@ public class KinesisClientLibConfiguration {
*/
// CHECKSTYLE:IGNORE HiddenFieldCheck FOR NEXT 26 LINES
// CHECKSTYLE:IGNORE ParameterNumber FOR NEXT 26 LINES
public KinesisClientLibConfiguration(String applicationName, String streamName, String kinesisEndpoint,
InitialPositionInStream initialPositionInStream, AwsCredentialsProvider kinesisCredentialsProvider,
AwsCredentialsProvider dynamoDBCredentialsProvider, AwsCredentialsProvider cloudWatchCredentialsProvider,
long failoverTimeMillis, String workerId, int maxRecords, long idleTimeBetweenReadsInMillis,
boolean callProcessRecordsEvenForEmptyRecordList, long parentShardPollIntervalMillis,
long shardSyncIntervalMillis, boolean cleanupTerminatedShardsBeforeExpiry, long taskBackoffTimeMillis,
long metricsBufferTimeMillis, int metricsMaxQueueSize, boolean validateSequenceNumberBeforeCheckpointing,
String regionName, long shutdownGraceMillis, long schedulerInitializationBackoffTimeMillis) {
this(applicationName, streamName, kinesisEndpoint, null, initialPositionInStream, kinesisCredentialsProvider,
dynamoDBCredentialsProvider, cloudWatchCredentialsProvider, failoverTimeMillis, workerId, maxRecords,
idleTimeBetweenReadsInMillis, callProcessRecordsEvenForEmptyRecordList, parentShardPollIntervalMillis,
shardSyncIntervalMillis, cleanupTerminatedShardsBeforeExpiry, taskBackoffTimeMillis,
metricsBufferTimeMillis, metricsMaxQueueSize, validateSequenceNumberBeforeCheckpointing, regionName,
shutdownGraceMillis, schedulerInitializationBackoffTimeMillis);
public KinesisClientLibConfiguration(
String applicationName,
String streamName,
String kinesisEndpoint,
InitialPositionInStream initialPositionInStream,
AwsCredentialsProvider kinesisCredentialsProvider,
AwsCredentialsProvider dynamoDBCredentialsProvider,
AwsCredentialsProvider cloudWatchCredentialsProvider,
long failoverTimeMillis,
String workerId,
int maxRecords,
long idleTimeBetweenReadsInMillis,
boolean callProcessRecordsEvenForEmptyRecordList,
long parentShardPollIntervalMillis,
long shardSyncIntervalMillis,
boolean cleanupTerminatedShardsBeforeExpiry,
long taskBackoffTimeMillis,
long metricsBufferTimeMillis,
int metricsMaxQueueSize,
boolean validateSequenceNumberBeforeCheckpointing,
String regionName,
long shutdownGraceMillis,
long schedulerInitializationBackoffTimeMillis) {
this(
applicationName,
streamName,
kinesisEndpoint,
null,
initialPositionInStream,
kinesisCredentialsProvider,
dynamoDBCredentialsProvider,
cloudWatchCredentialsProvider,
failoverTimeMillis,
workerId,
maxRecords,
idleTimeBetweenReadsInMillis,
callProcessRecordsEvenForEmptyRecordList,
parentShardPollIntervalMillis,
shardSyncIntervalMillis,
cleanupTerminatedShardsBeforeExpiry,
taskBackoffTimeMillis,
metricsBufferTimeMillis,
metricsMaxQueueSize,
validateSequenceNumberBeforeCheckpointing,
regionName,
shutdownGraceMillis,
schedulerInitializationBackoffTimeMillis);
}
/**
@ -448,15 +504,30 @@ public class KinesisClientLibConfiguration {
*/
// CHECKSTYLE:IGNORE HiddenFieldCheck FOR NEXT 26 LINES
// CHECKSTYLE:IGNORE ParameterNumber FOR NEXT 26 LINES
public KinesisClientLibConfiguration(String applicationName, String streamName, String kinesisEndpoint,
String dynamoDBEndpoint, InitialPositionInStream initialPositionInStream,
AwsCredentialsProvider kinesisCredentialsProvider, AwsCredentialsProvider dynamoDBCredentialsProvider,
AwsCredentialsProvider cloudWatchCredentialsProvider, long failoverTimeMillis, String workerId,
int maxRecords, long idleTimeBetweenReadsInMillis, boolean callProcessRecordsEvenForEmptyRecordList,
long parentShardPollIntervalMillis, long shardSyncIntervalMillis,
boolean cleanupTerminatedShardsBeforeExpiry, long taskBackoffTimeMillis, long metricsBufferTimeMillis,
int metricsMaxQueueSize, boolean validateSequenceNumberBeforeCheckpointing, String regionName,
long shutdownGraceMillis, long schedulerInitializationBackoffTimeMillis) {
public KinesisClientLibConfiguration(
String applicationName,
String streamName,
String kinesisEndpoint,
String dynamoDBEndpoint,
InitialPositionInStream initialPositionInStream,
AwsCredentialsProvider kinesisCredentialsProvider,
AwsCredentialsProvider dynamoDBCredentialsProvider,
AwsCredentialsProvider cloudWatchCredentialsProvider,
long failoverTimeMillis,
String workerId,
int maxRecords,
long idleTimeBetweenReadsInMillis,
boolean callProcessRecordsEvenForEmptyRecordList,
long parentShardPollIntervalMillis,
long shardSyncIntervalMillis,
boolean cleanupTerminatedShardsBeforeExpiry,
long taskBackoffTimeMillis,
long metricsBufferTimeMillis,
int metricsMaxQueueSize,
boolean validateSequenceNumberBeforeCheckpointing,
String regionName,
long shutdownGraceMillis,
long schedulerInitializationBackoffTimeMillis) {
// Check following values are greater than zero
checkIsValuePositive("FailoverTimeMillis", failoverTimeMillis);
checkIsValuePositive("IdleTimeBetweenReadsInMillis", idleTimeBetweenReadsInMillis);
@ -494,8 +565,8 @@ public class KinesisClientLibConfiguration {
this.maxLeasesToStealAtOneTime = DEFAULT_MAX_LEASES_TO_STEAL_AT_ONE_TIME;
this.initialLeaseTableReadCapacity = DEFAULT_INITIAL_LEASE_TABLE_READ_CAPACITY;
this.initialLeaseTableWriteCapacity = DEFAULT_INITIAL_LEASE_TABLE_WRITE_CAPACITY;
this.initialPositionInStreamExtended = InitialPositionInStreamExtended
.newInitialPosition(initialPositionInStream);
this.initialPositionInStreamExtended =
InitialPositionInStreamExtended.newInitialPosition(initialPositionInStream);
this.skipShardSyncAtWorkerInitializationIfLeasesExist = DEFAULT_SKIP_SHARD_SYNC_AT_STARTUP_IF_LEASES_EXIST;
this.shardPrioritization = DEFAULT_SHARD_PRIORITIZATION;
this.recordsFetcherFactory = new SimpleRecordsFetcherFactory();
@ -558,15 +629,30 @@ public class KinesisClientLibConfiguration {
*/
// CHECKSTYLE:IGNORE HiddenFieldCheck FOR NEXT 26 LINES
// CHECKSTYLE:IGNORE ParameterNumber FOR NEXT 26 LINES
public KinesisClientLibConfiguration(String applicationName, String streamName, String kinesisEndpoint,
String dynamoDBEndpoint, InitialPositionInStream initialPositionInStream,
AwsCredentialsProvider kinesisCredentialsProvider, AwsCredentialsProvider dynamoDBCredentialsProvider,
AwsCredentialsProvider cloudWatchCredentialsProvider, long failoverTimeMillis, String workerId,
int maxRecords, long idleTimeBetweenReadsInMillis, boolean callProcessRecordsEvenForEmptyRecordList,
long parentShardPollIntervalMillis, long shardSyncIntervalMillis,
boolean cleanupTerminatedShardsBeforeExpiry, long taskBackoffTimeMillis, long metricsBufferTimeMillis,
int metricsMaxQueueSize, boolean validateSequenceNumberBeforeCheckpointing, String regionName,
RecordsFetcherFactory recordsFetcherFactory, long schedulerInitializationBackoffTimeMillis) {
public KinesisClientLibConfiguration(
String applicationName,
String streamName,
String kinesisEndpoint,
String dynamoDBEndpoint,
InitialPositionInStream initialPositionInStream,
AwsCredentialsProvider kinesisCredentialsProvider,
AwsCredentialsProvider dynamoDBCredentialsProvider,
AwsCredentialsProvider cloudWatchCredentialsProvider,
long failoverTimeMillis,
String workerId,
int maxRecords,
long idleTimeBetweenReadsInMillis,
boolean callProcessRecordsEvenForEmptyRecordList,
long parentShardPollIntervalMillis,
long shardSyncIntervalMillis,
boolean cleanupTerminatedShardsBeforeExpiry,
long taskBackoffTimeMillis,
long metricsBufferTimeMillis,
int metricsMaxQueueSize,
boolean validateSequenceNumberBeforeCheckpointing,
String regionName,
RecordsFetcherFactory recordsFetcherFactory,
long schedulerInitializationBackoffTimeMillis) {
// Check following values are greater than zero
checkIsValuePositive("FailoverTimeMillis", failoverTimeMillis);
checkIsValuePositive("IdleTimeBetweenReadsInMillis", idleTimeBetweenReadsInMillis);
@ -606,8 +692,8 @@ public class KinesisClientLibConfiguration {
this.maxLeasesToStealAtOneTime = DEFAULT_MAX_LEASES_TO_STEAL_AT_ONE_TIME;
this.initialLeaseTableReadCapacity = DEFAULT_INITIAL_LEASE_TABLE_READ_CAPACITY;
this.initialLeaseTableWriteCapacity = DEFAULT_INITIAL_LEASE_TABLE_WRITE_CAPACITY;
this.initialPositionInStreamExtended = InitialPositionInStreamExtended
.newInitialPosition(initialPositionInStream);
this.initialPositionInStreamExtended =
InitialPositionInStreamExtended.newInitialPosition(initialPositionInStream);
this.skipShardSyncAtWorkerInitializationIfLeasesExist = DEFAULT_SKIP_SHARD_SYNC_AT_STARTUP_IF_LEASES_EXIST;
this.shardPrioritization = DEFAULT_SHARD_PRIORITIZATION;
this.recordsFetcherFactory = recordsFetcherFactory;
@ -932,8 +1018,8 @@ public class KinesisClientLibConfiguration {
*/
public KinesisClientLibConfiguration withInitialPositionInStream(InitialPositionInStream initialPositionInStream) {
this.initialPositionInStream = initialPositionInStream;
this.initialPositionInStreamExtended = InitialPositionInStreamExtended
.newInitialPosition(initialPositionInStream);
this.initialPositionInStreamExtended =
InitialPositionInStreamExtended.newInitialPosition(initialPositionInStream);
return this;
}
@ -959,6 +1045,11 @@ public class KinesisClientLibConfiguration {
return this;
}
public KinesisClientLibConfiguration withEnablePriorityLeaseAssignment(boolean enablePriorityLeaseAssignment) {
this.enablePriorityLeaseAssignment = enablePriorityLeaseAssignment;
return this;
}
/**
* @param shardSyncIntervalMillis
* Time between tasks to sync leases and Kinesis shards
@ -977,6 +1068,10 @@ public class KinesisClientLibConfiguration {
*/
public KinesisClientLibConfiguration withMaxRecords(int maxRecords) {
checkIsValuePositive("MaxRecords", (long) maxRecords);
if (maxRecords > DEFAULT_MAX_RECORDS) {
throw new IllegalArgumentException("maxRecords must be less than or equal to " + DEFAULT_MAX_RECORDS
+ " but current value is " + maxRecords);
}
this.maxRecords = maxRecords;
return this;
}
@ -1135,8 +1230,10 @@ public class KinesisClientLibConfiguration {
} else if (metricsEnabledDimensions.contains(MetricsScope.METRICS_DIMENSIONS_ALL)) {
this.metricsEnabledDimensions = METRICS_DIMENSIONS_ALL;
} else {
this.metricsEnabledDimensions = ImmutableSet.<String> builder().addAll(metricsEnabledDimensions)
.addAll(METRICS_ALWAYS_ENABLED_DIMENSIONS).build();
this.metricsEnabledDimensions = ImmutableSet.<String>builder()
.addAll(metricsEnabledDimensions)
.addAll(METRICS_ALWAYS_ENABLED_DIMENSIONS)
.build();
}
return this;
}
@ -1267,7 +1364,8 @@ public class KinesisClientLibConfiguration {
* @return this configuration object
*/
public KinesisClientLibConfiguration withMaxLeaseRenewalThreads(int maxLeaseRenewalThreads) {
Validate.isTrue(maxLeaseRenewalThreads > 2,
Validate.isTrue(
maxLeaseRenewalThreads > 2,
"The maximum number of lease renewal threads must be greater than or equal to 2.");
this.maxLeaseRenewalThreads = maxLeaseRenewalThreads;
@ -1327,7 +1425,8 @@ public class KinesisClientLibConfiguration {
* @return KinesisClientLibConfiguration
*/
public KinesisClientLibConfiguration withDataFetchingStrategy(String dataFetchingStrategy) {
this.recordsFetcherFactory.dataFetchingStrategy(DataFetchingStrategy.valueOf(dataFetchingStrategy.toUpperCase()));
this.recordsFetcherFactory.dataFetchingStrategy(
DataFetchingStrategy.valueOf(dataFetchingStrategy.toUpperCase()));
return this;
}
@ -1413,7 +1512,8 @@ public class KinesisClientLibConfiguration {
* Interval in milliseconds between retrying the scheduler initialization.
* @return
*/
public KinesisClientLibConfiguration withSchedulerInitializationBackoffTimeMillis(long schedulerInitializationBackoffTimeMillis) {
public KinesisClientLibConfiguration withSchedulerInitializationBackoffTimeMillis(
long schedulerInitializationBackoffTimeMillis) {
checkIsValuePositive("schedulerInitializationBackoffTimeMillis", schedulerInitializationBackoffTimeMillis);
this.schedulerInitializationBackoffTimeMillis = schedulerInitializationBackoffTimeMillis;
return this;

3
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/DrainChildSTDERRTask.java
View file

@ -23,8 +23,7 @@ import lombok.extern.slf4j.Slf4j;
*/
@Slf4j
class DrainChildSTDERRTask extends LineReaderTask<Boolean> {
DrainChildSTDERRTask() {
}
DrainChildSTDERRTask() {}
@Override
protected HandleLineResult<Boolean> handleLine(String line) {

9
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/DrainChildSTDOUTTask.java
View file

@ -22,23 +22,22 @@ import lombok.extern.slf4j.Slf4j;
* This class is used to drain the STDOUT of the child process. After the child process has been given a shutdown
* message and responded indicating that it is shutdown, we attempt to close the input and outputs of that process so
* that the process can exit.
*
*
* To understand why this is necessary, consider the following scenario:
*
*
* <ol>
* <li>Child process responds that it is done with shutdown.</li>
* <li>Child process prints debugging text to STDOUT that fills the pipe buffer so child becomes blocked.</li>
* <li>Parent process doesn't drain child process's STDOUT.</li>
* <li>Child process remains blocked.</li>
* </ol>
*
*
* To prevent the child process from becoming blocked in this way, it is the responsibility of the parent process to
* drain the child process's STDOUT. We reprint each drained line to our log to permit debugging.
*/
@Slf4j
class DrainChildSTDOUTTask extends LineReaderTask<Boolean> {
DrainChildSTDOUTTask() {
}
DrainChildSTDOUTTask() {}
@Override
protected HandleLineResult<Boolean> handleLine(String line) {

13
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/GetNextMessageTask.java
View file

@ -17,10 +17,9 @@ package software.amazon.kinesis.multilang;
import java.io.BufferedReader;
import java.io.IOException;
import software.amazon.kinesis.multilang.messages.Message;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.slf4j.Slf4j;
import software.amazon.kinesis.multilang.messages.Message;
/**
* Gets the next message off the STDOUT of the child process. Throws an exception if a message is not found before the
@ -34,7 +33,7 @@ class GetNextMessageTask extends LineReaderTask<Message> {
/**
* Constructor.
*
*
* @param objectMapper An object mapper for decoding json messages from the input stream.
*/
GetNextMessageTask(ObjectMapper objectMapper) {
@ -43,7 +42,7 @@ class GetNextMessageTask extends LineReaderTask<Message> {
/**
* Checks if a line is an empty line.
*
*
* @param line A string
* @return True if the line is an empty string, i.e. "", false otherwise.
*/
@ -71,8 +70,10 @@ class GetNextMessageTask extends LineReaderTask<Message> {
@Override
protected Message returnAfterException(Exception e) {
throw new RuntimeException("Encountered an error while reading a line from STDIN for shard " + getShardId()
+ " so won't be able to return a message.", e);
throw new RuntimeException(
"Encountered an error while reading a line from STDIN for shard " + getShardId()
+ " so won't be able to return a message.",
e);
}
@Override

20
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/LineReaderTask.java
View file

@ -30,7 +30,7 @@ import lombok.extern.slf4j.Slf4j;
* <li> {@link #returnAfterEndOfInput()}</li>
* <li> {@link #returnAfterException(Exception)}</li>
* </ol>
*
*
* @param <T>
*/
@Slf4j
@ -41,8 +41,7 @@ abstract class LineReaderTask<T> implements Callable<T> {
private String shardId;
LineReaderTask() {
}
LineReaderTask() {}
/**
* Reads lines off the input stream until a return value is set, or an exception is encountered, or the end of the
@ -72,7 +71,7 @@ abstract class LineReaderTask<T> implements Callable<T> {
* return from the {@link #call()} function by having a value, indicating that value should be returned immediately
* without reading further, or not having a value, indicating that more lines of input need to be read before
* returning.
*
*
* @param line A line read from the input stream.
* @return HandleLineResult<T> which may or may not have a has return value, indicating to return or not return yet
* respectively.
@ -83,7 +82,7 @@ abstract class LineReaderTask<T> implements Callable<T> {
* This method will be called if there is an error while reading from the input stream. The return value of this
* method will be returned as the result of this Callable unless an Exception is thrown. If an Exception is thrown
* then that exception will be thrown by the Callable.
*
*
* @param e An exception that occurred while reading from the input stream.
* @return What to return.
*/
@ -93,7 +92,7 @@ abstract class LineReaderTask<T> implements Callable<T> {
* This method will be called once the end of the input stream is reached. The return value of this method will be
* returned as the result of this Callable. Implementations of this method are welcome to throw a runtime exception
* to indicate that the task was unsuccessful.
*
*
* @return What to return.
*/
protected abstract T returnAfterEndOfInput();
@ -101,7 +100,7 @@ abstract class LineReaderTask<T> implements Callable<T> {
/**
* Allows subclasses to provide more detailed logs. Specifically, this allows the drain tasks and GetNextMessageTask
* to log which shard they're working on.
*
*
* @return The shard id
*/
public String getShardId() {
@ -110,7 +109,7 @@ abstract class LineReaderTask<T> implements Callable<T> {
/**
* The description should be a string explaining what this particular LineReader class does.
*
*
* @return The description.
*/
public String getDescription() {
@ -121,7 +120,7 @@ abstract class LineReaderTask<T> implements Callable<T> {
* The result of a call to {@link LineReaderTask#handleLine(String)}. Allows implementations of that method to
* indicate whether a particular invocation of that method produced a return for this task or not. If a return value
* doesn't exist the {@link #call()} method will continue to the next line.
*
*
* @param <V>
*/
protected class HandleLineResult<V> {
@ -158,7 +157,7 @@ abstract class LineReaderTask<T> implements Callable<T> {
* {@link MultiLangShardRecordProcessor#initialize(String)} is called. So we follow a pattern where the attributes are
* set inside this method instead of the constructor so that this object will be initialized when all its attributes
* are known to the record processor.
*
*
* @param stream
* @param shardId
* @param description
@ -180,5 +179,4 @@ abstract class LineReaderTask<T> implements Callable<T> {
this.description = description;
return this;
}
}

34
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MessageReader.java
View file

@ -20,19 +20,19 @@ import java.io.InputStreamReader;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import software.amazon.kinesis.multilang.messages.Message;
import com.fasterxml.jackson.databind.ObjectMapper;
import software.amazon.kinesis.multilang.messages.Message;
/**
* Provides methods for interacting with the child process's STDOUT.
*
*
* {@link #getNextMessageFromSTDOUT()} reads lines from the child process's STDOUT and attempts to decode a
* {@link Message} object from each line. A child process's STDOUT could have lines that don't contain data related to
* the multi-language protocol, such as when the child process prints debugging information to its STDOUT (instead of
* logging to a file), also when a child processes writes a Message it is expected to prepend and append a new line
* character to their message to help ensure that it is isolated on a line all by itself which results in empty lines
* being present in STDOUT. Lines which cannot be decoded to a Message object are ignored.
*
*
* {@link #drainSTDOUT()} simply reads all data from the child process's STDOUT until the stream is closed.
*/
class MessageReader {
@ -48,19 +48,18 @@ class MessageReader {
/**
* Use the initialize methods after construction.
*/
MessageReader() {
}
MessageReader() {}
/**
* Returns a future which represents an attempt to read the next message in the child process's STDOUT. If the task
* is successful, the result of the future will be the next message found in the child process's STDOUT, if the task
* is unable to find a message before the child process's STDOUT is closed, or reading from STDOUT causes an
* IOException, then an execution exception will be generated by this future.
*
*
* The task employed by this method reads from the child process's STDOUT line by line. The task attempts to decode
* each line into a {@link Message} object. Lines that fail to decode to a Message are ignored and the task
* continues to the next line until it finds a Message.
*
*
* @return
*/
Future<Message> getNextMessageFromSTDOUT() {
@ -73,7 +72,7 @@ class MessageReader {
* Returns a future that represents a computation that drains the STDOUT of the child process. That future's result
* is true if the end of the child's STDOUT is reached, its result is false if there was an error while reading from
* the stream. This task will log all the lines it drains to permit debugging.
*
*
* @return
*/
Future<Boolean> drainSTDOUT() {
@ -89,19 +88,16 @@ class MessageReader {
* {@link MultiLangShardRecordProcessor#initialize(String)} is called. So we follow a pattern where the attributes are
* set inside this method instead of the constructor so that this object will be initialized when all its attributes
* are known to the record processor.
*
*
* @param stream Used to read messages from the subprocess.
* @param shardId The shard we're working on.
* @param objectMapper The object mapper to decode messages.
* @param executorService An executor service to run tasks in.
*/
MessageReader initialize(InputStream stream,
String shardId,
ObjectMapper objectMapper,
ExecutorService executorService) {
return this.initialize(new BufferedReader(new InputStreamReader(stream)), shardId, objectMapper,
executorService);
MessageReader initialize(
InputStream stream, String shardId, ObjectMapper objectMapper, ExecutorService executorService) {
return this.initialize(
new BufferedReader(new InputStreamReader(stream)), shardId, objectMapper, executorService);
}
/**
@ -110,10 +106,8 @@ class MessageReader {
* @param objectMapper The object mapper to decode messages.
* @param executorService An executor service to run tasks in.
*/
MessageReader initialize(BufferedReader reader,
String shardId,
ObjectMapper objectMapper,
ExecutorService executorService) {
MessageReader initialize(
BufferedReader reader, String shardId, ObjectMapper objectMapper, ExecutorService executorService) {
this.reader = reader;
this.shardId = shardId;
this.objectMapper = objectMapper;

54
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MessageWriter.java
View file

@ -23,7 +23,6 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.slf4j.Slf4j;
import software.amazon.kinesis.lifecycle.events.InitializationInput;
import software.amazon.kinesis.lifecycle.events.LeaseLostInput;
@ -55,13 +54,12 @@ class MessageWriter {
/**
* Use initialize method after construction.
*/
MessageWriter() {
}
MessageWriter() {}
/**
* Writes the message then writes the line separator provided by the system. Flushes each message to guarantee it
* is delivered as soon as possible to the subprocess.
*
*
* @param message A message to be written to the subprocess.
* @return
* @throws IOException
@ -76,7 +74,10 @@ class MessageWriter {
*/
synchronized (writer) {
writer.write(message, 0, message.length());
writer.write(System.lineSeparator(), 0, System.lineSeparator().length());
writer.write(
System.lineSeparator(),
0,
System.lineSeparator().length());
writer.flush();
}
log.info("Message size == {} bytes for shard {}", message.getBytes().length, shardId);
@ -98,7 +99,7 @@ class MessageWriter {
/**
* Converts the message to a JSON string and writes it to the subprocess.
*
*
* @param message A message to be written to the subprocess.
* @return
*/
@ -108,9 +109,9 @@ class MessageWriter {
String jsonText = objectMapper.writeValueAsString(message);
return writeMessageToOutput(jsonText);
} catch (IOException e) {
String errorMessage =
String.format("Encountered I/O error while writing %s action to subprocess", message.getClass()
.getSimpleName());
String errorMessage = String.format(
"Encountered I/O error while writing %s action to subprocess",
message.getClass().getSimpleName());
log.error(errorMessage, e);
throw new RuntimeException(errorMessage, e);
}
@ -118,7 +119,7 @@ class MessageWriter {
/**
* Writes an {@link InitializeMessage} to the subprocess.
*
*
* @param initializationInput
* contains information about the shard being initialized
*/
@ -128,7 +129,7 @@ class MessageWriter {
/**
* Writes a {@link ProcessRecordsMessage} message to the subprocess.
*
*
* @param processRecordsInput
* the records, and associated metadata to be processed.
*/
@ -138,7 +139,7 @@ class MessageWriter {
/**
* Writes the lease lost message to the sub process.
*
*
* @param leaseLostInput
* the lease lost input. This is currently unused as lease loss doesn't actually have anything in it
* @return A future that is set when the message has been written.
@ -149,7 +150,7 @@ class MessageWriter {
/**
* Writes a message to the sub process indicating that the shard has ended
*
*
* @param shardEndedInput
* the shard end input. This is currently unused as the checkpoint is extracted, and used by the caller.
* @return A future that is set when the message has been written.
@ -167,7 +168,7 @@ class MessageWriter {
/**
* Writes a {@link CheckpointMessage} to the subprocess.
*
*
* @param sequenceNumber
* The sequence number that was checkpointed.
* @param subSequenceNumber
@ -175,14 +176,14 @@ class MessageWriter {
* @param throwable
* The exception that was thrown by a checkpoint attempt. Null if one didn't occur.
*/
Future<Boolean> writeCheckpointMessageWithError(String sequenceNumber, Long subSequenceNumber,
Throwable throwable) {
Future<Boolean> writeCheckpointMessageWithError(
String sequenceNumber, Long subSequenceNumber, Throwable throwable) {
return writeMessage(new CheckpointMessage(sequenceNumber, subSequenceNumber, throwable));
}
/**
* Closes the output stream and prevents further attempts to write.
*
*
* @throws IOException Thrown when closing the writer fails
*/
void close() throws IOException {
@ -201,18 +202,16 @@ class MessageWriter {
* {@link MultiLangShardRecordProcessor (String)} is called. So we follow a pattern where the attributes are
* set inside this method instead of the constructor so that this object will be initialized when all its attributes
* are known to the record processor.
*
*
* @param stream Used to write messages to the subprocess.
* @param shardId The shard we're working on.
* @param objectMapper The object mapper to encode messages.
* @param executorService An executor service to run tasks in.
*/
MessageWriter initialize(OutputStream stream,
String shardId,
ObjectMapper objectMapper,
ExecutorService executorService) {
return this.initialize(new BufferedWriter(new OutputStreamWriter(stream)), shardId, objectMapper,
executorService);
MessageWriter initialize(
OutputStream stream, String shardId, ObjectMapper objectMapper, ExecutorService executorService) {
return this.initialize(
new BufferedWriter(new OutputStreamWriter(stream)), shardId, objectMapper, executorService);
}
/**
@ -221,15 +220,12 @@ class MessageWriter {
* @param objectMapper The object mapper to encode messages.
* @param executorService An executor service to run tasks in.
*/
MessageWriter initialize(BufferedWriter writer,
String shardId,
ObjectMapper objectMapper,
ExecutorService executorService) {
MessageWriter initialize(
BufferedWriter writer, String shardId, ObjectMapper objectMapper, ExecutorService executorService) {
this.writer = writer;
this.shardId = shardId;
this.objectMapper = objectMapper;
this.executorService = executorService;
return this;
}
}

39
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MultiLangDaemon.java
View file

@ -26,20 +26,18 @@ import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.classic.joran.JoranConfigurator;
import ch.qos.logback.core.joran.spi.JoranException;
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
import lombok.Data;
import lombok.experimental.Accessors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.LoggerFactory;
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.classic.joran.JoranConfigurator;
import ch.qos.logback.core.joran.spi.JoranException;
import lombok.Data;
import lombok.experimental.Accessors;
import lombok.extern.slf4j.Slf4j;
import software.amazon.kinesis.coordinator.Scheduler;
/**
@ -75,11 +73,14 @@ public class MultiLangDaemon {
@Parameter
List<String> parameters = new ArrayList<>();
@Parameter(names = { "-p", "--properties-file" }, description = "Properties file to be used with the KCL")
@Parameter(
names = {"-p", "--properties-file"},
description = "Properties file to be used with the KCL")
String propertiesFile;
@Parameter(names = { "-l",
"--log-configuration" }, description = "File location of logback.xml to be override the default")
@Parameter(
names = {"-l", "--log-configuration"},
description = "File location of logback.xml to be override the default")
String logConfiguration;
}
@ -102,7 +103,8 @@ public class MultiLangDaemon {
}
JCommander buildJCommanderAndParseArgs(final MultiLangDaemonArguments arguments, final String[] args) {
JCommander jCommander = JCommander.newBuilder().programName("amazon-kinesis-client MultiLangDaemon")
JCommander jCommander = JCommander.newBuilder()
.programName("amazon-kinesis-client MultiLangDaemon")
.addObject(arguments)
.build();
jCommander.parse(args);
@ -128,8 +130,8 @@ public class MultiLangDaemon {
}
}
void configureLogging(final String logConfiguration, final LoggerContext loggerContext,
final JoranConfigurator configurator) {
void configureLogging(
final String logConfiguration, final LoggerContext loggerContext, final JoranConfigurator configurator) {
loggerContext.reset();
try (InputStream inputStream = FileUtils.openInputStream(new File(logConfiguration))) {
configurator.setContext(loggerContext);
@ -146,9 +148,8 @@ public class MultiLangDaemon {
if (arguments.parameters.size() == 1) {
propertiesFile = arguments.parameters.get(0);
} else {
throw new RuntimeException(
"Expected a single argument, but found multiple arguments. Arguments: "
+ String.join(", ", arguments.parameters));
throw new RuntimeException("Expected a single argument, but found multiple arguments. Arguments: "
+ String.join(", ", arguments.parameters));
}
}

53
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MultiLangDaemonConfig.java
View file

@ -26,10 +26,9 @@ import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import software.amazon.kinesis.multilang.config.KinesisClientLibConfigurator;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import lombok.extern.slf4j.Slf4j;
import software.amazon.kinesis.multilang.config.KinesisClientLibConfigurator;
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
import software.amazon.kinesis.retrieval.RetrievalConfig;
@ -45,15 +44,15 @@ public class MultiLangDaemonConfig {
private static final String PROP_PROCESSING_LANGUAGE = "processingLanguage";
private static final String PROP_MAX_ACTIVE_THREADS = "maxActiveThreads";
private MultiLangDaemonConfiguration multiLangDaemonConfiguration;
private final MultiLangDaemonConfiguration multiLangDaemonConfiguration;
private ExecutorService executorService;
private final ExecutorService executorService;
private MultiLangRecordProcessorFactory recordProcessorFactory;
private final MultiLangRecordProcessorFactory recordProcessorFactory;
/**
* Constructor.
*
*
* @param propertiesFile
* The location of the properties file.
* @throws IOException
@ -66,7 +65,7 @@ public class MultiLangDaemonConfig {
}
/**
*
*
* @param propertiesFile
* The location of the properties file.
* @param classLoader
@ -82,7 +81,7 @@ public class MultiLangDaemonConfig {
}
/**
*
*
* @param propertiesFile
* The location of the properties file.
* @param classLoader
@ -94,8 +93,9 @@ public class MultiLangDaemonConfig {
* @throws IllegalArgumentException
* Thrown when the contents of the properties file are not as expected.
*/
public MultiLangDaemonConfig(String propertiesFile, ClassLoader classLoader,
KinesisClientLibConfigurator configurator) throws IOException, IllegalArgumentException {
public MultiLangDaemonConfig(
String propertiesFile, ClassLoader classLoader, KinesisClientLibConfigurator configurator)
throws IOException, IllegalArgumentException {
Properties properties = loadProperties(classLoader, propertiesFile);
if (!validateProperties(properties)) {
throw new IllegalArgumentException(
@ -107,11 +107,14 @@ public class MultiLangDaemonConfig {
multiLangDaemonConfiguration = configurator.getConfiguration(properties);
executorService = buildExecutorService(properties);
recordProcessorFactory = new MultiLangRecordProcessorFactory(executableName, executorService,
multiLangDaemonConfiguration);
recordProcessorFactory =
new MultiLangRecordProcessorFactory(executableName, executorService, multiLangDaemonConfiguration);
log.info("Running {} to process stream {} with executable {}", multiLangDaemonConfiguration.getApplicationName(),
multiLangDaemonConfiguration.getStreamName(), executableName);
log.info(
"Running {} to process stream {} with executable {}",
multiLangDaemonConfiguration.getApplicationName(),
multiLangDaemonConfiguration.getStreamName(),
executableName);
prepare(processingLanguage);
}
@ -138,7 +141,7 @@ public class MultiLangDaemonConfig {
}
log.info("MultiLangDaemon is adding the following fields to the User Agent: {}", userAgent.toString());
// multiLangDaemonConfiguration.withUserAgent(userAgent.toString());
// multiLangDaemonConfiguration.withUserAgent(userAgent.toString());
}
private static Properties loadProperties(ClassLoader classLoader, String propertiesFileName) throws IOException {
@ -165,7 +168,6 @@ public class MultiLangDaemonConfig {
propertyStream.close();
}
}
}
private static boolean validateProperties(Properties properties) {
@ -182,17 +184,22 @@ public class MultiLangDaemonConfig {
log.debug("Value for {} property is {}", PROP_MAX_ACTIVE_THREADS, maxActiveThreads);
if (maxActiveThreads <= 0) {
log.info("Using a cached thread pool.");
return new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
builder.build());
return new ThreadPoolExecutor(
0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, new SynchronousQueue<>(), builder.build());
} else {
log.info("Using a fixed thread pool with {} max active threads.", maxActiveThreads);
return new ThreadPoolExecutor(maxActiveThreads, maxActiveThreads, 0L, TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<Runnable>(), builder.build());
return new ThreadPoolExecutor(
maxActiveThreads,
maxActiveThreads,
0L,
TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<>(),
builder.build());
}
}
/**
*
*
* @return A KinesisClientLibConfiguration object based on the properties file provided.
*/
public MultiLangDaemonConfiguration getMultiLangDaemonConfiguration() {
@ -200,7 +207,7 @@ public class MultiLangDaemonConfig {
}
/**
*
*
* @return An executor service based on the properties file provided.
*/
public ExecutorService getExecutorService() {
@ -208,7 +215,7 @@ public class MultiLangDaemonConfig {
}
/**
*
*
* @return A MultiLangRecordProcessorFactory based on the properties file provided.
*/
public MultiLangRecordProcessorFactory getRecordProcessorFactory() {

62
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MultiLangProtocol.java
View file

@ -61,8 +61,11 @@ class MultiLangProtocol {
* @param initializationInput
* information about the shard this processor is starting to process
*/
MultiLangProtocol(MessageReader messageReader, MessageWriter messageWriter,
InitializationInput initializationInput, MultiLangDaemonConfiguration configuration) {
MultiLangProtocol(
MessageReader messageReader,
MessageWriter messageWriter,
InitializationInput initializationInput,
MultiLangDaemonConfiguration configuration) {
this.messageReader = messageReader;
this.messageWriter = messageWriter;
this.initializationInput = initializationInput;
@ -82,7 +85,6 @@ class MultiLangProtocol {
*/
Future<Boolean> writeFuture = messageWriter.writeInitializeMessage(initializationInput);
return waitForStatusMessage(InitializeMessage.ACTION, null, writeFuture);
}
/**
@ -100,7 +102,7 @@ class MultiLangProtocol {
/**
* Notifies the client process that the lease has been lost, and it needs to shutdown.
*
*
* @param leaseLostInput
* the lease lost input that is passed to the {@link MessageWriter}
* @return true if the message was successfully writtem
@ -115,7 +117,9 @@ class MultiLangProtocol {
* @return
*/
boolean shardEnded(ShardEndedInput shardEndedInput) {
return waitForStatusMessage(ShardEndedMessage.ACTION, shardEndedInput.checkpointer(),
return waitForStatusMessage(
ShardEndedMessage.ACTION,
shardEndedInput.checkpointer(),
messageWriter.writeShardEndedMessage(shardEndedInput));
}
@ -147,8 +151,8 @@ class MultiLangProtocol {
* The writing task.
* @return Whether or not this operation succeeded.
*/
private boolean waitForStatusMessage(String action, RecordProcessorCheckpointer checkpointer,
Future<Boolean> writeFuture) {
private boolean waitForStatusMessage(
String action, RecordProcessorCheckpointer checkpointer, Future<Boolean> writeFuture) {
boolean statusWasCorrect = waitForStatusMessage(action, checkpointer);
// Examine whether or not we failed somewhere along the line.
@ -194,7 +198,7 @@ class MultiLangProtocol {
return false;
}
statusMessage = message.filter(m -> m instanceof StatusMessage).map(m -> (StatusMessage) m );
statusMessage = message.filter(m -> m instanceof StatusMessage).map(m -> (StatusMessage) m);
}
return this.validateStatusMessage(statusMessage.get(), action);
}
@ -207,13 +211,17 @@ class MultiLangProtocol {
try {
return Optional.of(fm.get());
} catch (InterruptedException e) {
log.error("Interrupted while waiting for {} message for shard {}", action,
initializationInput.shardId(), e);
log.error(
"Interrupted while waiting for {} message for shard {}", action, initializationInput.shardId(), e);
} catch (ExecutionException e) {
log.error("Failed to get status message for {} action for shard {}", action,
initializationInput.shardId(), e);
log.error(
"Failed to get status message for {} action for shard {}",
action,
initializationInput.shardId(),
e);
} catch (TimeoutException e) {
log.error("Timedout to get status message for {} action for shard {}. Terminating...",
log.error(
"Timedout to get status message for {} action for shard {}. Terminating...",
action,
initializationInput.shardId(),
e);
@ -240,11 +248,14 @@ class MultiLangProtocol {
* @return Whether or not this operation succeeded.
*/
private boolean validateStatusMessage(StatusMessage statusMessage, String action) {
log.info("Received response {} from subprocess while waiting for {}"
+ " while processing shard {}", statusMessage, action, initializationInput.shardId());
return !(statusMessage == null || statusMessage.getResponseFor() == null || !statusMessage.getResponseFor()
.equals(action));
log.info(
"Received response {} from subprocess while waiting for {}" + " while processing shard {}",
statusMessage,
action,
initializationInput.shardId());
return !(statusMessage == null
|| statusMessage.getResponseFor() == null
|| !statusMessage.getResponseFor().equals(action));
}
/**
@ -274,13 +285,12 @@ class MultiLangProtocol {
}
return this.messageWriter.writeCheckpointMessageWithError(sequenceNumber, subSequenceNumber, null);
} else {
String message =
String.format("Was asked to checkpoint at %s but no checkpointer was provided for shard %s",
sequenceNumber, initializationInput.shardId());
String message = String.format(
"Was asked to checkpoint at %s but no checkpointer was provided for shard %s",
sequenceNumber, initializationInput.shardId());
log.error(message);
return this.messageWriter.writeCheckpointMessageWithError(sequenceNumber, subSequenceNumber,
new InvalidStateException(
message));
return this.messageWriter.writeCheckpointMessageWithError(
sequenceNumber, subSequenceNumber, new InvalidStateException(message));
}
} catch (Throwable t) {
return this.messageWriter.writeCheckpointMessageWithError(sequenceNumber, subSequenceNumber, t);
@ -288,8 +298,8 @@ class MultiLangProtocol {
}
private String logCheckpointMessage(String sequenceNumber, Long subSequenceNumber) {
return String.format("Attempting to checkpoint shard %s @ sequence number %s, and sub sequence number %s",
return String.format(
"Attempting to checkpoint shard %s @ sequence number %s, and sub sequence number %s",
initializationInput.shardId(), sequenceNumber, subSequenceNumber);
}
}

19
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MultiLangRecordProcessorFactory.java
View file

@ -17,12 +17,10 @@ package software.amazon.kinesis.multilang;
import java.util.concurrent.ExecutorService;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.slf4j.Slf4j;
import software.amazon.kinesis.coordinator.KinesisClientLibConfiguration;
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
import software.amazon.kinesis.processor.ShardRecordProcessorFactory;
import software.amazon.kinesis.processor.ShardRecordProcessor;
import software.amazon.kinesis.processor.ShardRecordProcessorFactory;
/**
* Creates {@link MultiLangShardRecordProcessor}'s.
@ -44,8 +42,8 @@ public class MultiLangRecordProcessorFactory implements ShardRecordProcessorFact
* @param command The command that will do processing for this factory's record processors.
* @param executorService An executor service to use while processing inputs and outputs of the child process.
*/
public MultiLangRecordProcessorFactory(String command, ExecutorService executorService,
MultiLangDaemonConfiguration configuration) {
public MultiLangRecordProcessorFactory(
String command, ExecutorService executorService, MultiLangDaemonConfiguration configuration) {
this(command, executorService, new ObjectMapper(), configuration);
}
@ -54,8 +52,11 @@ public class MultiLangRecordProcessorFactory implements ShardRecordProcessorFact
* @param executorService An executor service to use while processing inputs and outputs of the child process.
* @param objectMapper An object mapper used to convert messages to json to be written to the child process
*/
public MultiLangRecordProcessorFactory(String command, ExecutorService executorService, ObjectMapper objectMapper,
MultiLangDaemonConfiguration configuration) {
public MultiLangRecordProcessorFactory(
String command,
ExecutorService executorService,
ObjectMapper objectMapper,
MultiLangDaemonConfiguration configuration) {
this.command = command;
this.commandArray = command.split(COMMAND_DELIMETER_REGEX);
this.executorService = executorService;
@ -69,8 +70,8 @@ public class MultiLangRecordProcessorFactory implements ShardRecordProcessorFact
/*
* Giving ProcessBuilder the command as an array of Strings allows users to specify command line arguments.
*/
return new MultiLangShardRecordProcessor(new ProcessBuilder(commandArray), executorService, this.objectMapper,
this.configuration);
return new MultiLangShardRecordProcessor(
new ProcessBuilder(commandArray), executorService, this.objectMapper, this.configuration);
}
String[] getCommandArray() {

66
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/MultiLangShardRecordProcessor.java
View file

@ -22,7 +22,6 @@ import java.util.concurrent.Future;
import java.util.function.Function;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.slf4j.Slf4j;
import software.amazon.kinesis.lifecycle.events.InitializationInput;
import software.amazon.kinesis.lifecycle.events.LeaseLostInput;
@ -32,7 +31,6 @@ import software.amazon.kinesis.lifecycle.events.ShutdownRequestedInput;
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
import software.amazon.kinesis.processor.ShardRecordProcessor;
/**
* A record processor that manages creating a child process that implements the multi language protocol and connecting
* that child process's input and outputs to a {@link MultiLangProtocol} object and calling the appropriate methods on
@ -50,20 +48,20 @@ public class MultiLangShardRecordProcessor implements ShardRecordProcessor {
private Future<?> stderrReadTask;
private MessageWriter messageWriter;
private MessageReader messageReader;
private DrainChildSTDERRTask readSTDERRTask;
private final MessageWriter messageWriter;
private final MessageReader messageReader;
private final DrainChildSTDERRTask readSTDERRTask;
private ProcessBuilder processBuilder;
private final ProcessBuilder processBuilder;
private Process process;
private ExecutorService executorService;
private final ExecutorService executorService;
private ProcessState state;
private ObjectMapper objectMapper;
private final ObjectMapper objectMapper;
private MultiLangProtocol protocol;
private MultiLangDaemonConfiguration configuration;
private final MultiLangDaemonConfiguration configuration;
@Override
public void initialize(InitializationInput initializationInput) {
@ -157,8 +155,10 @@ public class MultiLangShardRecordProcessor implements ShardRecordProcessor {
if (ProcessState.ACTIVE.equals(this.state)) {
stopProcessing("Encountered an error while trying to shutdown child process", t);
} else {
stopProcessing("Encountered an error during shutdown,"
+ " but it appears the processor has already been shutdown", t);
stopProcessing(
"Encountered an error during shutdown,"
+ " but it appears the processor has already been shutdown",
t);
}
}
}
@ -167,12 +167,13 @@ public class MultiLangShardRecordProcessor implements ShardRecordProcessor {
* Used to tell whether the processor has been shutdown already.
*/
private enum ProcessState {
ACTIVE, SHUTDOWN
ACTIVE,
SHUTDOWN
}
/**
* Constructor.
*
*
* @param processBuilder
* Provides process builder functionality.
* @param executorService
@ -180,15 +181,24 @@ public class MultiLangShardRecordProcessor implements ShardRecordProcessor {
* @param objectMapper
* An obejct mapper.
*/
MultiLangShardRecordProcessor(ProcessBuilder processBuilder, ExecutorService executorService,
ObjectMapper objectMapper, MultiLangDaemonConfiguration configuration) {
this(processBuilder, executorService, objectMapper, new MessageWriter(), new MessageReader(),
new DrainChildSTDERRTask(), configuration);
MultiLangShardRecordProcessor(
ProcessBuilder processBuilder,
ExecutorService executorService,
ObjectMapper objectMapper,
MultiLangDaemonConfiguration configuration) {
this(
processBuilder,
executorService,
objectMapper,
new MessageWriter(),
new MessageReader(),
new DrainChildSTDERRTask(),
configuration);
}
/**
* Note: This constructor has package level access solely for testing purposes.
*
*
* @param processBuilder
* Provides the child process for this record processor
* @param executorService
@ -202,9 +212,14 @@ public class MultiLangShardRecordProcessor implements ShardRecordProcessor {
* @param readSTDERRTask
* Error reader to read from child process's stderr
*/
MultiLangShardRecordProcessor(ProcessBuilder processBuilder, ExecutorService executorService, ObjectMapper objectMapper,
MessageWriter messageWriter, MessageReader messageReader, DrainChildSTDERRTask readSTDERRTask,
MultiLangDaemonConfiguration configuration) {
MultiLangShardRecordProcessor(
ProcessBuilder processBuilder,
ExecutorService executorService,
ObjectMapper objectMapper,
MessageWriter messageWriter,
MessageReader messageReader,
DrainChildSTDERRTask readSTDERRTask,
MultiLangDaemonConfiguration configuration) {
this.executorService = executorService;
this.processBuilder = processBuilder;
this.objectMapper = objectMapper;
@ -213,7 +228,6 @@ public class MultiLangShardRecordProcessor implements ShardRecordProcessor {
this.readSTDERRTask = readSTDERRTask;
this.configuration = configuration;
this.state = ProcessState.ACTIVE;
}
@ -270,7 +284,7 @@ public class MultiLangShardRecordProcessor implements ShardRecordProcessor {
/**
* Convenience method used by {@link #childProcessShutdownSequence()} to drain the STDIN and STDERR of the child
* process.
*
*
* @param future A future to wait on.
* @param whatThisFutureIsDoing What that future is doing while we wait.
*/
@ -285,7 +299,7 @@ public class MultiLangShardRecordProcessor implements ShardRecordProcessor {
/**
* Convenience method for logging and safely shutting down so that we don't throw an exception up to the KCL on
* accident.
*
*
* @param message The reason we are stopping processing.
* @param reason An exception that caused us to want to stop processing.
*/
@ -303,8 +317,6 @@ public class MultiLangShardRecordProcessor implements ShardRecordProcessor {
/**
* We provide a package level method for unit testing this call to exit.
*
* @param val exit value
*/
void exit() {
System.exit(EXIT_VALUE);
@ -313,7 +325,7 @@ public class MultiLangShardRecordProcessor implements ShardRecordProcessor {
/**
* The {@link ProcessBuilder} class is final so not easily mocked. We wrap the only interaction we have with it in
* this package level method to permit unit testing.
*
*
* @return The process started by processBuilder
* @throws IOException If the process can't be started.
*/

142
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/NestedPropertyKey.java Normal file
View file

@ -0,0 +1,142 @@
/*
* Copyright 2023 Amazon.com, Inc. or its affiliates.
* Licensed under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package software.amazon.kinesis.multilang;
import java.util.HashMap;
import java.util.Map;
import com.amazonaws.regions.Regions;
import com.google.common.base.CaseFormat;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
/**
* Key-Value pairs which may be nested in, and extracted from, a property value
* in a Java properties file. For example, given the line in a property file of
* {@code my_key = my_value|foo=bar} and a delimiter split on {@code |} (pipe),
* the value {@code my_value|foo=bar} would have a nested key of {@code foo}
* and its corresponding value is {@code bar}.
* <br/><br/>
* The order of nested properties does not matter, and these properties are optional.
* Customers may choose to provide, in any order, zero-or-more nested properties.
* <br/><br/>
* Duplicate keys are not supported, and may result in a last-write-wins outcome.
*/
@Slf4j
public enum NestedPropertyKey {
/**
* Specify the service endpoint where requests will be submitted.
* This property's value must be in the following format:
* <pre>
* ENDPOINT ::= SERVICE_ENDPOINT "^" SIGNING_REGION
* SERVICE_ENDPOINT ::= URL
* SIGNING_REGION ::= AWS_REGION
* </pre>
*
* It would be redundant to provide both this and {@link #ENDPOINT_REGION}.
*
* @see #ENDPOINT_REGION
* @see <a href="https://docs.aws.amazon.com/general/latest/gr/rande.html">AWS Service endpoints</a>
* @see <a href="https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-regions">Available Regions</a>
*/
ENDPOINT {
void visit(final NestedPropertyProcessor processor, final String endpoint) {
final String[] tokens = endpoint.split("\\^");
if (tokens.length != 2) {
throw new IllegalArgumentException("Invalid " + name() + ": " + endpoint);
}
processor.acceptEndpoint(tokens[0], tokens[1]);
}
},
/**
* Specify the region where service requests will be submitted. This
* region will determine both the service endpoint and signing region.
* <br/><br/>
* It would be redundant to provide both this and {@link #ENDPOINT}.
*
* @see #ENDPOINT
* @see <a href="https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-regions">Available Regions</a>
*/
ENDPOINT_REGION {
void visit(final NestedPropertyProcessor processor, final String region) {
processor.acceptEndpointRegion(Regions.fromName(region));
}
},
/**
* External ids may be used when delegating access in a multi-tenant
* environment, or to third parties.
*
* @see <a href="https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-user_externalid.html">
* How to use an external ID when granting access to your AWS resources to a third party</a>
*/
EXTERNAL_ID {
void visit(final NestedPropertyProcessor processor, final String externalId) {
processor.acceptExternalId(externalId);
}
},
;
/**
* Nested key within the property value. For example, a nested key-value
* of {@code foo=bar} has a nested key of {@code foo}.
*/
@Getter(AccessLevel.PACKAGE)
private final String nestedKey;
NestedPropertyKey() {
// convert the enum from UPPER_SNAKE_CASE to lowerCamelCase
nestedKey = CaseFormat.UPPER_UNDERSCORE.to(CaseFormat.LOWER_CAMEL, name());
}
abstract void visit(NestedPropertyProcessor processor, String value);
/**
* Parses any number of parameters. Each nested property will prompt a
* visit to the {@code processor}.
*
* @param processor processor to be invoked for every nested property
* @param params parameters to check for a nested property key
*/
public static void parse(final NestedPropertyProcessor processor, final String... params) {
// Construct a disposable cache to keep this O(n). Since parsing is
// usually one-and-done, it's wasteful to maintain this cache in perpetuity.
final Map<String, NestedPropertyKey> cachedKeys = new HashMap<>();
for (final NestedPropertyKey npk : values()) {
cachedKeys.put(npk.getNestedKey(), npk);
}
for (final String param : params) {
if (param != null) {
final String[] tokens = param.split("=");
if (tokens.length == 2) {
final NestedPropertyKey npk = cachedKeys.get(tokens[0]);
if (npk != null) {
npk.visit(processor, tokens[1]);
} else {
log.warn("Unsupported nested key: {}", param);
}
} else if (tokens.length > 2) {
log.warn("Malformed nested key: {}", param);
} else {
log.info("Parameter is not a nested key: {}", param);
}
}
}
}
}

53
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/NestedPropertyProcessor.java Normal file
View file

@ -0,0 +1,53 @@
/*
* Copyright 2023 Amazon.com, Inc. or its affiliates.
* Licensed under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package software.amazon.kinesis.multilang;
import com.amazonaws.regions.Regions;
/**
* Defines methods to process {@link NestedPropertyKey}s.
*/
public interface NestedPropertyProcessor {
/**
* Set the service endpoint where requests are sent.
*
* @param serviceEndpoint the service endpoint either with or without the protocol
* (e.g., https://sns.us-west-1.amazonaws.com, sns.us-west-1.amazonaws.com)
* @param signingRegion the region to use for SigV4 signing of requests (e.g. us-west-1)
*
* @see #acceptEndpointRegion(Regions)
* @see <a href="https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/client/builder/AwsClientBuilder.EndpointConfiguration.html">
* AwsClientBuilder.EndpointConfiguration</a>
*/
void acceptEndpoint(String serviceEndpoint, String signingRegion);
/**
* Set the service endpoint where requests are sent.
*
* @param region Region to be used by the client. This will be used to determine both the service endpoint
* (e.g., https://sns.us-west-1.amazonaws.com) and signing region (e.g., us-west-1) for requests.
*
* @see #acceptEndpoint(String, String)
*/
void acceptEndpointRegion(Regions region);
/**
* Set the external id, an optional field to designate who can assume an IAM role.
*
* @param externalId external id used in the service call used to retrieve session credentials
*/
void acceptExternalId(String externalId);
}

86
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/auth/KclSTSAssumeRoleSessionCredentialsProvider.java Normal file
View file

@ -0,0 +1,86 @@
/*
* Copyright 2023 Amazon.com, Inc. or its affiliates.
* Licensed under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package software.amazon.kinesis.multilang.auth;
import java.util.Arrays;
import com.amazonaws.auth.AWSSessionCredentials;
import com.amazonaws.auth.AWSSessionCredentialsProvider;
import com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider;
import com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider.Builder;
import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.securitytoken.AWSSecurityTokenService;
import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient;
import software.amazon.kinesis.multilang.NestedPropertyKey;
import software.amazon.kinesis.multilang.NestedPropertyProcessor;
/**
* An {@link AWSSessionCredentialsProvider} that is backed by STSAssumeRole.
*/
public class KclSTSAssumeRoleSessionCredentialsProvider
implements AWSSessionCredentialsProvider, NestedPropertyProcessor {
private final Builder builder;
private final STSAssumeRoleSessionCredentialsProvider provider;
/**
*
* @param params vararg parameters which must include roleArn at index=0,
* and roleSessionName at index=1
*/
public KclSTSAssumeRoleSessionCredentialsProvider(final String[] params) {
this(params[0], params[1], Arrays.copyOfRange(params, 2, params.length));
}
public KclSTSAssumeRoleSessionCredentialsProvider(
final String roleArn, final String roleSessionName, final String... params) {
builder = new Builder(roleArn, roleSessionName);
NestedPropertyKey.parse(this, params);
provider = builder.build();
}
@Override
public AWSSessionCredentials getCredentials() {
return provider.getCredentials();
}
@Override
public void refresh() {
// do nothing
}
@Override
public void acceptEndpoint(final String serviceEndpoint, final String signingRegion) {
final EndpointConfiguration endpoint = new EndpointConfiguration(serviceEndpoint, signingRegion);
final AWSSecurityTokenService stsClient = AWSSecurityTokenServiceClient.builder()
.withEndpointConfiguration(endpoint)
.build();
builder.withStsClient(stsClient);
}
@Override
public void acceptEndpointRegion(final Regions region) {
final AWSSecurityTokenService stsClient =
AWSSecurityTokenServiceClient.builder().withRegion(region).build();
builder.withStsClient(stsClient);
}
@Override
public void acceptExternalId(final String externalId) {
builder.withExternalId(externalId);
}
}

129
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/AWSCredentialsProviderPropertyValueDecoder.java
View file

@ -14,31 +14,30 @@
*/
package software.amazon.kinesis.multilang.config;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.AWSCredentialsProviderChain;
import lombok.extern.slf4j.Slf4j;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain;
/**
* Get AWSCredentialsProvider property.
*/
@Slf4j
class AWSCredentialsProviderPropertyValueDecoder implements IPropertyValueDecoder<AWSCredentialsProvider> {
private static final String AUTH_PREFIX = "com.amazonaws.auth.";
private static final String LIST_DELIMITER = ",";
private static final String ARG_DELIMITER = "|";
/**
* Constructor.
*/
AWSCredentialsProviderPropertyValueDecoder() {
}
AWSCredentialsProviderPropertyValueDecoder() {}
/**
* Get AWSCredentialsProvider property.
@ -65,35 +64,58 @@ class AWSCredentialsProviderPropertyValueDecoder implements IPropertyValueDecode
*/
@Override
public List<Class<AWSCredentialsProvider>> getSupportedTypes() {
return Arrays.asList(AWSCredentialsProvider.class);
return Collections.singletonList(AWSCredentialsProvider.class);
}
/*
/**
* Convert string list to a list of valid credentials providers.
*/
private static List<AWSCredentialsProvider> getValidCredentialsProviders(List<String> providerNames) {
List<AWSCredentialsProvider> credentialsProviders = new ArrayList<>();
for (String providerName : providerNames) {
if (providerName.contains(ARG_DELIMITER)) {
String[] nameAndArgs = providerName.split("\\" + ARG_DELIMITER);
Class<?>[] argTypes = new Class<?>[nameAndArgs.length - 1];
Arrays.fill(argTypes, String.class);
try {
Class<?> className = Class.forName(nameAndArgs[0]);
Constructor<?> c = className.getConstructor(argTypes);
credentialsProviders.add((AWSCredentialsProvider) c
.newInstance(Arrays.copyOfRange(nameAndArgs, 1, nameAndArgs.length)));
} catch (Exception e) {
log.debug("Can't find any credentials provider matching {}.", providerName);
final String[] nameAndArgs = providerName.split("\\" + ARG_DELIMITER);
final Class<? extends AWSCredentialsProvider> clazz;
try {
final Class<?> c = Class.forName(nameAndArgs[0]);
if (!AWSCredentialsProvider.class.isAssignableFrom(c)) {
continue;
}
} else {
try {
Class<?> className = Class.forName(providerName);
credentialsProviders.add((AWSCredentialsProvider) className.newInstance());
} catch (Exception e) {
log.debug("Can't find any credentials provider matching {}.", providerName);
clazz = (Class<? extends AWSCredentialsProvider>) c;
} catch (ClassNotFoundException cnfe) {
// Providers are a product of prefixed Strings to cover multiple
// namespaces (e.g., "Foo" -> { "some.auth.Foo", "kcl.auth.Foo" }).
// It's expected that many class names will not resolve.
continue;
}
log.info("Attempting to construct {}", clazz);
AWSCredentialsProvider provider = null;
if (nameAndArgs.length > 1) {
final String[] varargs = Arrays.copyOfRange(nameAndArgs, 1, nameAndArgs.length);
// attempt to invoke an explicit N-arg constructor of FooClass(String, String, ...)
provider = constructProvider(providerName, () -> {
Class<?>[] argTypes = new Class<?>[nameAndArgs.length - 1];
Arrays.fill(argTypes, String.class);
return clazz.getConstructor(argTypes).newInstance(varargs);
});
if (provider == null) {
// attempt to invoke a public varargs/array constructor of FooClass(String[])
provider = constructProvider(providerName, () -> clazz.getConstructor(String[].class)
.newInstance((Object) varargs));
}
}
if (provider == null) {
// regardless of parameters, fallback to invoke a public no-arg constructor
provider = constructProvider(providerName, clazz::newInstance);
}
if (provider != null) {
credentialsProviders.add(provider);
}
}
return credentialsProviders;
}
@ -101,7 +123,7 @@ class AWSCredentialsProviderPropertyValueDecoder implements IPropertyValueDecode
private static List<String> getProviderNames(String property) {
// assume list delimiter is ","
String[] elements = property.split(LIST_DELIMITER);
List<String> result = new ArrayList<String>();
List<String> result = new ArrayList<>();
for (int i = 0; i < elements.length; i++) {
String string = elements[i].trim();
if (!string.isEmpty()) {
@ -112,20 +134,49 @@ class AWSCredentialsProviderPropertyValueDecoder implements IPropertyValueDecode
return result;
}
private static List<String> getPossibleFullClassNames(String s) {
/*
* We take care of three cases :
*
* 1. Customer provides a short name of common providers in com.amazonaws.auth package i.e. any classes
* implementing the AWSCredentialsProvider interface:
* http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/AWSCredentialsProvider.html
*
* 2. Customer provides a full name of common providers e.g. com.amazonaws.auth.ClasspathFileCredentialsProvider
*
* 3. Customer provides a custom credentials provider with full name of provider
*/
private static List<String> getPossibleFullClassNames(final String provider) {
return Stream.of(
// Customer provides a short name of common providers in com.amazonaws.auth package
// (e.g., any classes implementing the AWSCredentialsProvider interface)
// @see
// http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/AWSCredentialsProvider.html
"com.amazonaws.auth.",
return Arrays.asList(s, AUTH_PREFIX + s);
// Customer provides a short name of a provider offered by this multi-lang package
"software.amazon.kinesis.multilang.auth.",
// Customer provides a fully-qualified provider name, or a custom credentials provider
// (e.g., com.amazonaws.auth.ClasspathFileCredentialsProvider, org.mycompany.FooProvider)
"")
.map(prefix -> prefix + provider)
.collect(Collectors.toList());
}
@FunctionalInterface
private interface CredentialsProviderConstructor<T extends AWSCredentialsProvider> {
T construct()
throws IllegalAccessException, InstantiationException, InvocationTargetException, NoSuchMethodException;
}
/**
* Attempts to construct an {@link AWSCredentialsProvider}.
*
* @param providerName Raw, unmodified provider name. Should there be an
* Exeception during construction, this parameter will be logged.
* @param constructor supplier-like function that will perform the construction
* @return the constructed provider, if successful; otherwise, null
*
* @param <T> type of the CredentialsProvider to construct
*/
private static <T extends AWSCredentialsProvider> T constructProvider(
final String providerName, final CredentialsProviderConstructor<T> constructor) {
try {
return constructor.construct();
} catch (NoSuchMethodException ignored) {
// ignore
} catch (IllegalAccessException | InstantiationException | InvocationTargetException | RuntimeException e) {
log.warn("Failed to construct {}", providerName, e);
}
return null;
}
}

30
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/BuilderDynaBean.java
View file

@ -20,7 +20,6 @@ import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.function.Function;
import java.util.function.Supplier;
import lombok.Getter;
import org.apache.commons.beanutils.ConvertUtilsBean;
@ -32,7 +31,7 @@ import org.apache.commons.lang3.StringUtils;
public class BuilderDynaBean implements DynaBean {
private static final String[] CLASS_NAME_JOINERS = { ClassUtils.PACKAGE_SEPARATOR, ClassUtils.INNER_CLASS_SEPARATOR };
private static final String[] CLASS_NAME_JOINERS = {ClassUtils.PACKAGE_SEPARATOR, ClassUtils.INNER_CLASS_SEPARATOR};
static final String NO_MAP_ACCESS_SUPPORT = "Map access isn't supported";
private Class<?> destinedClass;
@ -52,16 +51,22 @@ public class BuilderDynaBean implements DynaBean {
this(destinedClass, convertUtilsBean, null, Arrays.asList(classPrefixSearchList));
}
public BuilderDynaBean(Class<?> destinedClass, ConvertUtilsBean convertUtilsBean,
Function<String, ?> emptyPropertyHandler, String... classPrefixSearchList) {
public BuilderDynaBean(
Class<?> destinedClass,
ConvertUtilsBean convertUtilsBean,
Function<String, ?> emptyPropertyHandler,
String... classPrefixSearchList) {
this(destinedClass, convertUtilsBean, emptyPropertyHandler, Arrays.asList(classPrefixSearchList));
}
public BuilderDynaBean(Class<?> destinedClass, ConvertUtilsBean convertUtilsBean,
Function<String, ?> emtpyPropertyHandler, List<String> classPrefixSearchList) {
public BuilderDynaBean(
Class<?> destinedClass,
ConvertUtilsBean convertUtilsBean,
Function<String, ?> emptyPropertyHandler,
List<String> classPrefixSearchList) {
this.convertUtilsBean = convertUtilsBean;
this.classPrefixSearchList = classPrefixSearchList;
this.emptyPropertyHandler = emtpyPropertyHandler;
this.emptyPropertyHandler = emptyPropertyHandler;
initialize(destinedClass);
}
@ -103,7 +108,6 @@ public class BuilderDynaBean implements DynaBean {
// Ignored
//
}
}
}
}
@ -150,7 +154,6 @@ public class BuilderDynaBean implements DynaBean {
} else {
return expected.cast(dynaBeanCreateSupport.build());
}
}
private void validateResolvedEmptyHandler() {
@ -216,8 +219,10 @@ public class BuilderDynaBean implements DynaBean {
validateCanBuildOrCreate();
List<TypeTag> types = dynaBeanBuilderSupport.getProperty(name);
if (types.size() > 1) {
Optional<TypeTag> arrayType = types.stream().filter(t -> t.type.isArray()).findFirst();
return arrayType.map(t -> new DynaProperty(name, t.type, t.type.getComponentType()))
Optional<TypeTag> arrayType =
types.stream().filter(t -> t.type.isArray()).findFirst();
return arrayType
.map(t -> new DynaProperty(name, t.type, t.type.getComponentType()))
.orElseGet(() -> new DynaProperty(name));
} else {
TypeTag type = types.get(0);
@ -234,7 +239,8 @@ public class BuilderDynaBean implements DynaBean {
@Override
public DynaProperty[] getDynaProperties() {
validateCanBuildOrCreate();
return dynaBeanBuilderSupport.getPropertyNames().stream().map(this::getDynaProperty)
return dynaBeanBuilderSupport.getPropertyNames().stream()
.map(this::getDynaProperty)
.toArray(DynaProperty[]::new);
}

6
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/ConfigurationSettable.java
View file

@ -28,21 +28,21 @@ public @interface ConfigurationSettable {
/**
* Which builder this option applies to
*
*
* @return the class of the builder to use
*/
Class<?> configurationClass();
/**
* The method name on the builder, defaults to the fieldName
*
*
* @return the name of the method or null to use the default
*/
String methodName() default "";
/**
* If the type is actually an optional value this will enable conversions
*
*
* @return true if the value should be wrapped by an optional
*/
boolean convertToOptional() default false;

16
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/ConfigurationSettableUtils.java
View file

@ -23,13 +23,11 @@ import java.util.Map;
import java.util.Optional;
import java.util.Set;
import com.google.common.base.Defaults;
import lombok.NonNull;
import org.apache.commons.lang3.ClassUtils;
import org.apache.commons.lang3.StringUtils;
import com.google.common.base.Defaults;
import lombok.NonNull;
public class ConfigurationSettableUtils {
public static <T> T resolveFields(@NonNull Object source, @NonNull T configObject) {
@ -40,8 +38,8 @@ public class ConfigurationSettableUtils {
return configObject;
}
public static void resolveFields(Object source, Map<Class<?>, Object> configObjects, Set<Class<?>> restrictTo,
Set<Class<?>> skipIf) {
public static void resolveFields(
Object source, Map<Class<?>, Object> configObjects, Set<Class<?>> restrictTo, Set<Class<?>> skipIf) {
for (Field field : source.getClass().getDeclaredFields()) {
for (ConfigurationSettable b : field.getAnnotationsByType(ConfigurationSettable.class)) {
if (restrictTo != null && !restrictTo.contains(b.configurationClass())) {
@ -70,9 +68,11 @@ public class ConfigurationSettableUtils {
value = Optional.of(value);
}
if (ClassUtils.isPrimitiveOrWrapper(value.getClass())) {
Class<?> primitiveType = field.getType().isPrimitive() ? field.getType()
Class<?> primitiveType = field.getType().isPrimitive()
? field.getType()
: ClassUtils.wrapperToPrimitive(field.getType());
Class<?> wrapperType = !field.getType().isPrimitive() ? field.getType()
Class<?> wrapperType = !field.getType().isPrimitive()
? field.getType()
: ClassUtils.primitiveToWrapper(field.getType());
try {

4
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/DatePropertyValueDecoder.java
View file

@ -26,8 +26,7 @@ public class DatePropertyValueDecoder implements IPropertyValueDecoder<Date> {
/**
* Constructor.
*/
DatePropertyValueDecoder() {
}
DatePropertyValueDecoder() {}
/**
* @param value property value as String
@ -49,5 +48,4 @@ public class DatePropertyValueDecoder implements IPropertyValueDecoder<Date> {
public List<Class<Date>> getSupportedTypes() {
return Arrays.asList(Date.class);
}
}

47
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/DynaBeanBuilderSupport.java
View file

@ -29,11 +29,10 @@ import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Supplier;
import org.apache.commons.beanutils.ConvertUtilsBean;
import org.apache.commons.lang3.ClassUtils;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
import org.apache.commons.beanutils.ConvertUtilsBean;
import org.apache.commons.lang3.ClassUtils;
class DynaBeanBuilderSupport {
@ -48,8 +47,8 @@ class DynaBeanBuilderSupport {
private final Multimap<String, TypeTag> properties = HashMultimap.create();
private final Map<String, Object> values = new HashMap<>();
DynaBeanBuilderSupport(Class<?> destinedClass, ConvertUtilsBean convertUtilsBean,
List<String> classPrefixSearchList) {
DynaBeanBuilderSupport(
Class<?> destinedClass, ConvertUtilsBean convertUtilsBean, List<String> classPrefixSearchList) {
this.destinedClass = destinedClass;
this.convertUtilsBean = convertUtilsBean;
this.classPrefixSearchList = classPrefixSearchList;
@ -103,11 +102,12 @@ class DynaBeanBuilderSupport {
private Object createForProperty(String name) {
Optional<TypeTag> type = properties.get(name).stream().findFirst();
return type.map(t -> {
if (DynaBeanBuilderUtils.isBuilderOrCreate(t.type) || !t.hasConverter) {
return new BuilderDynaBean(t.type, convertUtilsBean, null, classPrefixSearchList);
}
return null;
}).orElse(null);
if (DynaBeanBuilderUtils.isBuilderOrCreate(t.type) || !t.hasConverter) {
return new BuilderDynaBean(t.type, convertUtilsBean, null, classPrefixSearchList);
}
return null;
})
.orElse(null);
}
boolean hasValue(String name) {
@ -157,8 +157,11 @@ class DynaBeanBuilderSupport {
void set(String name, Object value) {
if (value instanceof String && properties.get(name).stream().anyMatch(t -> t.type.isEnum())) {
TypeTag typeTag = properties.get(name).stream().filter(t -> t.type.isEnum()).findFirst().orElseThrow(
() -> new IllegalStateException("Expected enum type for " + name + ", but couldn't find it."));
TypeTag typeTag = properties.get(name).stream()
.filter(t -> t.type.isEnum())
.findFirst()
.orElseThrow(() ->
new IllegalStateException("Expected enum type for " + name + ", but couldn't find it."));
Class<? extends Enum> enumClass = (Class<? extends Enum>) typeTag.type;
values.put(name, Enum.valueOf(enumClass, value.toString()));
} else {
@ -174,9 +177,11 @@ class DynaBeanBuilderSupport {
private Object getArgument(Map.Entry<String, Object> setValue) {
Object argument = setValue.getValue();
if (argument instanceof Object[]) {
TypeTag arrayType = properties.get(setValue.getKey()).stream().filter(t -> t.type.isArray()).findFirst()
.orElseThrow(() -> new IllegalStateException(String
.format("Received Object[] for %s but can't find corresponding type", setValue.getKey())));
TypeTag arrayType = properties.get(setValue.getKey()).stream()
.filter(t -> t.type.isArray())
.findFirst()
.orElseThrow(() -> new IllegalStateException(String.format(
"Received Object[] for %s but can't find corresponding type", setValue.getKey())));
Object[] arrayValues = (Object[]) argument;
Object[] destination = (Object[]) Array.newInstance(arrayType.type.getComponentType(), arrayValues.length);
@ -212,10 +217,12 @@ class DynaBeanBuilderSupport {
for (Map.Entry<String, Object> setValue : values.entrySet()) {
Object argument = getArgument(setValue);
Method mutator = properties.get(setValue.getKey()).stream()
.filter(t -> ClassUtils.isAssignable(argument.getClass(), t.type)).findFirst()
.map(a -> a.builderMethod).orElseThrow(
() -> new IllegalStateException(String.format("Unable to find mutator for %s of type %s",
setValue.getKey(), argument.getClass().getName())));
.filter(t -> ClassUtils.isAssignable(argument.getClass(), t.type))
.findFirst()
.map(a -> a.builderMethod)
.orElseThrow(() -> new IllegalStateException(String.format(
"Unable to find mutator for %s of type %s",
setValue.getKey(), argument.getClass().getName())));
try {
source = mutator.invoke(source, argument);
} catch (IllegalAccessException | InvocationTargetException e) {
@ -236,7 +243,6 @@ class DynaBeanBuilderSupport {
} catch (IllegalAccessException | NoSuchMethodException | InvocationTargetException e) {
throw new RuntimeException(e);
}
}
Collection<String> getPropertyNames() {
@ -249,5 +255,4 @@ class DynaBeanBuilderSupport {
}
return new ArrayList<>(properties.get(name));
}
}

20
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/DynaBeanCreateSupport.java
View file

@ -30,8 +30,8 @@ class DynaBeanCreateSupport {
private final List<TypeTag> createTypes = new ArrayList<>();
private Object[] createValues = null;
DynaBeanCreateSupport(Class<?> destinedClass, ConvertUtilsBean convertUtilsBean,
List<String> classPrefixSearchList) {
DynaBeanCreateSupport(
Class<?> destinedClass, ConvertUtilsBean convertUtilsBean, List<String> classPrefixSearchList) {
this.destinedClass = destinedClass;
this.convertUtilsBean = convertUtilsBean;
this.classPrefixSearchList = classPrefixSearchList;
@ -58,8 +58,8 @@ class DynaBeanCreateSupport {
Object build() {
Method createMethod = DynaBeanBuilderUtils.getMethod(destinedClass, "create",
createTypes.stream().map(t -> t.type).toArray(i -> new Class<?>[i]));
Method createMethod = DynaBeanBuilderUtils.getMethod(
destinedClass, "create", createTypes.stream().map(t -> t.type).toArray(i -> new Class<?>[i]));
Object arguments[] = new Object[createValues.length];
for (int i = 0; i < createValues.length; ++i) {
if (createValues[i] instanceof BuilderDynaBean) {
@ -77,8 +77,8 @@ class DynaBeanCreateSupport {
return createValues[index];
} else {
if (createValues[index] == null) {
createValues[index] = new BuilderDynaBean(createTypes.get(index).type, convertUtilsBean, null,
classPrefixSearchList);
createValues[index] = new BuilderDynaBean(
createTypes.get(index).type, convertUtilsBean, null, classPrefixSearchList);
}
return createValues[index];
}
@ -89,13 +89,11 @@ class DynaBeanCreateSupport {
public void set(String name, int index, Object value) {
if (StringUtils.isEmpty(name)) {
if (index >= createValues.length) {
throw new IllegalArgumentException(
String.format("%d exceeds the maximum number of arguments (%d) for %s", index,
createValues.length, destinedClass.getName()));
throw new IllegalArgumentException(String.format(
"%d exceeds the maximum number of arguments (%d) for %s",
index, createValues.length, destinedClass.getName()));
}
createValues[index] = value;
}
}
}

13
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/FanoutConfigBean.java
View file

@ -26,21 +26,28 @@ public class FanoutConfigBean implements RetrievalConfigBuilder {
@ConfigurationSettable(configurationClass = FanOutConfig.class)
private int maxDescribeStreamSummaryRetries;
@ConfigurationSettable(configurationClass = FanOutConfig.class)
private String consumerArn;
@ConfigurationSettable(configurationClass = FanOutConfig.class)
private String consumerName;
@ConfigurationSettable(configurationClass = FanOutConfig.class)
private int maxDescribeStreamConsumerRetries;
@ConfigurationSettable(configurationClass = FanOutConfig.class)
private int registerStreamConsumerRetries;
@ConfigurationSettable(configurationClass = FanOutConfig.class)
private long retryBackoffMillis;
@Override
public FanOutConfig build(KinesisAsyncClient kinesisAsyncClient, MultiLangDaemonConfiguration parent) {
return ConfigurationSettableUtils.resolveFields(this, new FanOutConfig(kinesisAsyncClient).applicationName(parent.getApplicationName())
.streamName(parent.getStreamName()));
return ConfigurationSettableUtils.resolveFields(
this,
new FanOutConfig(kinesisAsyncClient)
.applicationName(parent.getApplicationName())
.streamName(parent.getStreamName()));
}
}

6
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/IPropertyValueDecoder.java
View file

@ -18,13 +18,13 @@ import java.util.List;
/**
* This class captures the concept of decoding a property value to a particular Java type.
*
*
* @param <T>
*/
interface IPropertyValueDecoder<T> {
/**
* Get the value that was read from a configuration file and convert it to some type.
*
*
* @param propertyValue property string value that needs to be decoded.
* @return property value in type T
*/
@ -32,7 +32,7 @@ interface IPropertyValueDecoder<T> {
/**
* Get a list of supported types for this class.
*
*
* @return list of supported classes.
*/
List<Class<T>> getSupportedTypes();

3
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/IntegerPropertyValueDecoder.java
View file

@ -25,8 +25,7 @@ class IntegerPropertyValueDecoder implements IPropertyValueDecoder<Integer> {
/**
* Constructor.
*/
IntegerPropertyValueDecoder() {
}
IntegerPropertyValueDecoder() {}
/**
* @param value property value as String

31
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/KinesisClientLibConfigurator.java
View file

@ -19,12 +19,12 @@ import java.io.InputStream;
import java.lang.reflect.InvocationTargetException;
import java.util.Properties;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.beanutils.BeanUtilsBean;
import org.apache.commons.beanutils.ConvertUtilsBean;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Validate;
import software.amazon.awssdk.arns.Arn;
import software.amazon.kinesis.common.StreamIdentifier;
/**
* KinesisClientLibConfigurator constructs a KinesisClientLibConfiguration from java properties file. The following
@ -40,7 +40,6 @@ public class KinesisClientLibConfigurator {
private final BeanUtilsBean utilsBean;
private final MultiLangDaemonConfiguration configuration;
/**
* Constructor.
*/
@ -55,13 +54,14 @@ public class KinesisClientLibConfigurator {
* Program will fail immediately, if customer provide: 1) invalid variable value. Program will log it as warning and
* continue, if customer provide: 1) variable with unsupported variable type. 2) a variable with name which does not
* match any of the variables in KinesisClientLibConfigration.
*
*
* @param properties a Properties object containing the configuration information
* @return KinesisClientLibConfiguration
*/
public MultiLangDaemonConfiguration getConfiguration(Properties properties) {
properties.entrySet().forEach(e -> {
try {
log.info("Processing (key={}, value={})", e.getKey(), e.getValue());
utilsBean.setProperty(configuration, (String) e.getKey(), e.getValue());
} catch (IllegalAccessException | InvocationTargetException ex) {
throw new RuntimeException(ex);
@ -69,8 +69,23 @@ public class KinesisClientLibConfigurator {
});
Validate.notBlank(configuration.getApplicationName(), "Application name is required");
Validate.notBlank(configuration.getStreamName(), "Stream name is required");
Validate.isTrue(configuration.getKinesisCredentialsProvider().isDirty(), "A basic set of AWS credentials must be provided");
if (configuration.getStreamArn() != null
&& !configuration.getStreamArn().trim().isEmpty()) {
final Arn streamArnObj = Arn.fromString(configuration.getStreamArn());
StreamIdentifier.validateArn(streamArnObj);
// Parse out the stream Name from the Arn (and/or override existing value for Stream Name)
final String streamNameFromArn = streamArnObj.resource().resource();
configuration.setStreamName(streamNameFromArn);
}
Validate.notBlank(
configuration.getStreamName(),
"Stream name or Stream Arn is required. Stream Arn takes precedence if both are passed in.");
Validate.isTrue(
configuration.getKinesisCredentialsProvider().isDirty(),
"A basic set of AWS credentials must be provided");
return configuration;
}
@ -95,6 +110,4 @@ public class KinesisClientLibConfigurator {
}
return getConfiguration(properties);
}
}

206
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/MultiLangDaemonConfiguration.java
View file

@ -27,18 +27,16 @@ import java.util.Set;
import java.util.UUID;
import java.util.function.Function;
import org.apache.commons.beanutils.BeanUtilsBean;
import org.apache.commons.beanutils.ConvertUtils;
import org.apache.commons.beanutils.ConvertUtilsBean;
import org.apache.commons.beanutils.Converter;
import org.apache.commons.beanutils.converters.ArrayConverter;
import org.apache.commons.beanutils.converters.StringConverter;
import lombok.Data;
import lombok.Getter;
import lombok.Setter;
import lombok.experimental.Delegate;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.beanutils.BeanUtilsBean;
import org.apache.commons.beanutils.ConvertUtilsBean;
import org.apache.commons.beanutils.Converter;
import org.apache.commons.beanutils.converters.ArrayConverter;
import org.apache.commons.beanutils.converters.StringConverter;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient;
@ -73,6 +71,7 @@ public class MultiLangDaemonConfiguration {
private String applicationName;
private String streamName;
private String streamArn;
@ConfigurationSettable(configurationClass = ConfigsBuilder.class)
private String tableName;
@ -85,20 +84,37 @@ public class MultiLangDaemonConfiguration {
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private long failoverTimeMillis;
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private Boolean enablePriorityLeaseAssignment;
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private Boolean leaseTableDeletionProtectionEnabled;
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private Boolean leaseTablePitrEnabled;
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private long shardSyncIntervalMillis;
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private boolean cleanupLeasesUponShardCompletion;
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private boolean ignoreUnexpectedChildShards;
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private int maxLeasesForWorker;
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private int maxLeasesToStealAtOneTime;
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private int initialLeaseTableReadCapacity;
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private int initialLeaseTableWriteCapacity;
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class, methodName = "initialPositionInStream")
@ConfigurationSettable(configurationClass = RetrievalConfig.class)
private InitialPositionInStreamExtended initialPositionInStreamExtended;
@ -111,14 +127,16 @@ public class MultiLangDaemonConfiguration {
}
public void setInitialPositionInStream(InitialPositionInStream initialPositionInStream) {
this.initialPositionInStreamExtended = InitialPositionInStreamExtended
.newInitialPosition(initialPositionInStream);
this.initialPositionInStreamExtended =
InitialPositionInStreamExtended.newInitialPosition(initialPositionInStream);
}
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private int maxLeaseRenewalThreads;
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private long listShardsBackoffTimeInMillis;
@ConfigurationSettable(configurationClass = LeaseManagementConfig.class)
private int maxListShardsRetryAttempts;
@ -128,10 +146,13 @@ public class MultiLangDaemonConfiguration {
@ConfigurationSettable(configurationClass = CoordinatorConfig.class)
private long parentShardPollIntervalMillis;
@ConfigurationSettable(configurationClass = CoordinatorConfig.class)
private ShardPrioritization shardPrioritization;
@ConfigurationSettable(configurationClass = CoordinatorConfig.class)
private boolean skipShardSyncAtWorkerInitializationIfLeasesExist;
@ConfigurationSettable(configurationClass = CoordinatorConfig.class)
private long schedulerInitializationBackoffTimeMillis;
@ -140,12 +161,16 @@ public class MultiLangDaemonConfiguration {
@ConfigurationSettable(configurationClass = MetricsConfig.class)
private long metricsBufferTimeMillis;
@ConfigurationSettable(configurationClass = MetricsConfig.class)
private int metricsMaxQueueSize;
@ConfigurationSettable(configurationClass = MetricsConfig.class)
private MetricsLevel metricsLevel;
@ConfigurationSettable(configurationClass = LifecycleConfig.class, convertToOptional = true)
private Long logWarningForTaskAfterMillis;
@ConfigurationSettable(configurationClass = MetricsConfig.class)
private Set<String> metricsEnabledDimensions;
@ -157,10 +182,10 @@ public class MultiLangDaemonConfiguration {
metricsEnabledDimensions = new HashSet<>(Arrays.asList(dimensions));
}
private RetrievalMode retrievalMode = RetrievalMode.DEFAULT;
private final FanoutConfigBean fanoutConfig = new FanoutConfigBean();
@Delegate(types = PollingConfigBean.PollingConfigBeanDelegate.class)
private final PollingConfigBean pollingConfig = new PollingConfigBean();
@ -169,7 +194,6 @@ public class MultiLangDaemonConfiguration {
private long shutdownGraceMillis;
private Integer timeoutInSeconds;
private final BuilderDynaBean kinesisCredentialsProvider;
public void setAWSCredentialsProvider(String providerString) {
@ -199,61 +223,75 @@ public class MultiLangDaemonConfiguration {
this.utilsBean = utilsBean;
this.convertUtilsBean = convertUtilsBean;
convertUtilsBean.register(new Converter() {
@Override
public <T> T convert(Class<T> type, Object value) {
Date date = new Date(Long.parseLong(value.toString()) * 1000L);
return type.cast(InitialPositionInStreamExtended.newInitialPositionAtTimestamp(date));
}
}, InitialPositionInStreamExtended.class);
convertUtilsBean.register(
new Converter() {
@Override
public <T> T convert(Class<T> type, Object value) {
Date date = new Date(Long.parseLong(value.toString()) * 1000L);
return type.cast(InitialPositionInStreamExtended.newInitialPositionAtTimestamp(date));
}
},
InitialPositionInStreamExtended.class);
convertUtilsBean.register(new Converter() {
@Override
public <T> T convert(Class<T> type, Object value) {
return type.cast(MetricsLevel.valueOf(value.toString().toUpperCase()));
}
}, MetricsLevel.class);
convertUtilsBean.register(
new Converter() {
@Override
public <T> T convert(Class<T> type, Object value) {
return type.cast(MetricsLevel.valueOf(value.toString().toUpperCase()));
}
},
MetricsLevel.class);
convertUtilsBean.register(new Converter() {
@Override
public <T> T convert(Class<T> type, Object value) {
return type.cast(InitialPositionInStream.valueOf(value.toString().toUpperCase()));
}
}, InitialPositionInStream.class);
convertUtilsBean.register(
new Converter() {
@Override
public <T> T convert(Class<T> type, Object value) {
return type.cast(
InitialPositionInStream.valueOf(value.toString().toUpperCase()));
}
},
InitialPositionInStream.class);
convertUtilsBean.register(new Converter() {
@Override
public <T> T convert(Class<T> type, Object value) {
return type.cast(URI.create(value.toString()));
}
}, URI.class);
convertUtilsBean.register(
new Converter() {
@Override
public <T> T convert(Class<T> type, Object value) {
return type.cast(URI.create(value.toString()));
}
},
URI.class);
convertUtilsBean.register(new Converter() {
@Override
public <T> T convert(Class<T> type, Object value) {
return type.cast(RetrievalMode.from(value.toString()));
}
}, RetrievalMode.class);
convertUtilsBean.register(
new Converter() {
@Override
public <T> T convert(Class<T> type, Object value) {
return type.cast(RetrievalMode.from(value.toString()));
}
},
RetrievalMode.class);
convertUtilsBean.register(new Converter() {
@Override
public <T> T convert(final Class<T> type, final Object value) {
return type.cast(Region.of(value.toString()));
}
}, Region.class);
convertUtilsBean.register(
new Converter() {
@Override
public <T> T convert(final Class<T> type, final Object value) {
return type.cast(Region.of(value.toString()));
}
},
Region.class);
ArrayConverter arrayConverter = new ArrayConverter(String[].class, new StringConverter());
arrayConverter.setDelimiter(',');
convertUtilsBean.register(arrayConverter, String[].class);
AWSCredentialsProviderPropertyValueDecoder oldCredentialsDecoder = new AWSCredentialsProviderPropertyValueDecoder();
AWSCredentialsProviderPropertyValueDecoder oldCredentialsDecoder =
new AWSCredentialsProviderPropertyValueDecoder();
Function<String, ?> converter = s -> new V2CredentialWrapper(oldCredentialsDecoder.decodeValue(s));
this.kinesisCredentialsProvider = new BuilderDynaBean(AwsCredentialsProvider.class, convertUtilsBean,
converter, CREDENTIALS_DEFAULT_SEARCH_PATH);
this.dynamoDBCredentialsProvider = new BuilderDynaBean(AwsCredentialsProvider.class, convertUtilsBean,
converter, CREDENTIALS_DEFAULT_SEARCH_PATH);
this.cloudWatchCredentialsProvider = new BuilderDynaBean(AwsCredentialsProvider.class, convertUtilsBean,
converter, CREDENTIALS_DEFAULT_SEARCH_PATH);
this.kinesisCredentialsProvider = new BuilderDynaBean(
AwsCredentialsProvider.class, convertUtilsBean, converter, CREDENTIALS_DEFAULT_SEARCH_PATH);
this.dynamoDBCredentialsProvider = new BuilderDynaBean(
AwsCredentialsProvider.class, convertUtilsBean, converter, CREDENTIALS_DEFAULT_SEARCH_PATH);
this.cloudWatchCredentialsProvider = new BuilderDynaBean(
AwsCredentialsProvider.class, convertUtilsBean, converter, CREDENTIALS_DEFAULT_SEARCH_PATH);
this.kinesisClient = new BuilderDynaBean(KinesisAsyncClient.class, convertUtilsBean);
this.dynamoDbClient = new BuilderDynaBean(DynamoDbAsyncClient.class, convertUtilsBean);
@ -299,8 +337,8 @@ public class MultiLangDaemonConfiguration {
return credsBuilder.build(AwsCredentialsProvider.class);
}
private void updateCredentials(BuilderDynaBean toUpdate, AwsCredentialsProvider primary,
AwsCredentialsProvider secondary) {
private void updateCredentials(
BuilderDynaBean toUpdate, AwsCredentialsProvider primary, AwsCredentialsProvider secondary) {
if (toUpdate.hasValue("credentialsProvider")) {
return;
@ -328,8 +366,8 @@ public class MultiLangDaemonConfiguration {
}
private void handleRetrievalConfig(RetrievalConfig retrievalConfig, ConfigsBuilder configsBuilder) {
retrievalConfig
.retrievalSpecificConfig(retrievalMode.builder(this).build(configsBuilder.kinesisClient(), this));
retrievalConfig.retrievalSpecificConfig(
retrievalMode.builder(this).build(configsBuilder.kinesisClient(), this));
}
private Object adjustKinesisHttpConfiguration(Object builderObj) {
@ -352,8 +390,14 @@ public class MultiLangDaemonConfiguration {
final RetrievalConfig retrievalConfig;
public Scheduler build() {
return new Scheduler(checkpointConfig, coordinatorConfig, leaseManagementConfig, lifecycleConfig,
metricsConfig, processorConfig, retrievalConfig);
return new Scheduler(
checkpointConfig,
coordinatorConfig,
leaseManagementConfig,
lifecycleConfig,
metricsConfig,
processorConfig,
retrievalConfig);
}
}
@ -366,19 +410,25 @@ public class MultiLangDaemonConfiguration {
updateCredentials(dynamoDbClient, dynamoDbCreds, kinesisCreds);
updateCredentials(cloudWatchClient, cloudwatchCreds, kinesisCreds);
KinesisAsyncClient kinesisAsyncClient = kinesisClient.build(KinesisAsyncClient.class,
this::adjustKinesisHttpConfiguration);
KinesisAsyncClient kinesisAsyncClient =
kinesisClient.build(KinesisAsyncClient.class, this::adjustKinesisHttpConfiguration);
DynamoDbAsyncClient dynamoDbAsyncClient = dynamoDbClient.build(DynamoDbAsyncClient.class);
CloudWatchAsyncClient cloudWatchAsyncClient = cloudWatchClient.build(CloudWatchAsyncClient.class);
ConfigsBuilder configsBuilder = new ConfigsBuilder(streamName, applicationName, kinesisAsyncClient,
dynamoDbAsyncClient, cloudWatchAsyncClient, workerIdentifier, shardRecordProcessorFactory);
ConfigsBuilder configsBuilder = new ConfigsBuilder(
streamName,
applicationName,
kinesisAsyncClient,
dynamoDbAsyncClient,
cloudWatchAsyncClient,
workerIdentifier,
shardRecordProcessorFactory);
Map<Class<?>, Object> configObjects = new HashMap<>();
addConfigObjects(configObjects, configsBuilder);
resolveFields(configObjects, Collections.singleton(ConfigsBuilder.class),
Collections.singleton(PollingConfig.class));
resolveFields(
configObjects, Collections.singleton(ConfigsBuilder.class), Collections.singleton(PollingConfig.class));
CoordinatorConfig coordinatorConfig = configsBuilder.coordinatorConfig();
CheckpointConfig checkpointConfig = configsBuilder.checkpointConfig();
@ -388,19 +438,31 @@ public class MultiLangDaemonConfiguration {
ProcessorConfig processorConfig = configsBuilder.processorConfig();
RetrievalConfig retrievalConfig = configsBuilder.retrievalConfig();
addConfigObjects(configObjects, coordinatorConfig, checkpointConfig, leaseManagementConfig, lifecycleConfig,
metricsConfig, processorConfig, retrievalConfig);
addConfigObjects(
configObjects,
coordinatorConfig,
checkpointConfig,
leaseManagementConfig,
lifecycleConfig,
metricsConfig,
processorConfig,
retrievalConfig);
handleRetrievalConfig(retrievalConfig, configsBuilder);
resolveFields(configObjects, null, new HashSet<>(Arrays.asList(ConfigsBuilder.class, PollingConfig.class)));
return new ResolvedConfiguration(coordinatorConfig, checkpointConfig, leaseManagementConfig, lifecycleConfig,
metricsConfig, processorConfig, retrievalConfig);
return new ResolvedConfiguration(
coordinatorConfig,
checkpointConfig,
leaseManagementConfig,
lifecycleConfig,
metricsConfig,
processorConfig,
retrievalConfig);
}
public Scheduler build(ShardRecordProcessorFactory shardRecordProcessorFactory) {
return resolvedConfiguration(shardRecordProcessorFactory).build();
}
}

16
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/PollingConfigBean.java
View file

@ -30,34 +30,44 @@ public class PollingConfigBean implements RetrievalConfigBuilder {
interface PollingConfigBeanDelegate {
Integer getRetryGetRecordsInSeconds();
void setRetryGetRecordsInSeconds(Integer value);
Integer getMaxGetRecordsThreadPool();
void setMaxGetRecordsThreadPool(Integer value);
long getIdleTimeBetweenReadsInMillis();
void setIdleTimeBetweenReadsInMillis(long value);
int getMaxRecords();
void setMaxRecords(int value);
}
@ConfigurationSettable(configurationClass = PollingConfig.class, convertToOptional = true)
private Integer retryGetRecordsInSeconds;
@ConfigurationSettable(configurationClass = PollingConfig.class, convertToOptional = true)
private Integer maxGetRecordsThreadPool;
@ConfigurationSettable(configurationClass = PollingConfig.class)
private long idleTimeBetweenReadsInMillis;
@ConfigurationSettable(configurationClass = PollingConfig.class)
private int maxRecords;
public boolean anyPropertiesSet() {
return retryGetRecordsInSeconds != null || maxGetRecordsThreadPool != null || idleTimeBetweenReadsInMillis != 0 || maxRecords != 0;
return retryGetRecordsInSeconds != null
|| maxGetRecordsThreadPool != null
|| idleTimeBetweenReadsInMillis != 0
|| maxRecords != 0;
}
@Override
public PollingConfig build(KinesisAsyncClient kinesisAsyncClient, MultiLangDaemonConfiguration parent) {
return ConfigurationSettableUtils.resolveFields(this, new PollingConfig(parent.getStreamName(), kinesisAsyncClient));
return ConfigurationSettableUtils.resolveFields(
this, new PollingConfig(parent.getStreamName(), kinesisAsyncClient));
}
}

2
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/RetrievalConfigBuilder.java
View file

@ -21,7 +21,7 @@ import software.amazon.kinesis.retrieval.RetrievalSpecificConfig;
public interface RetrievalConfigBuilder {
/**
* Creates a retrieval specific configuration using the supplied parameters, and internal class parameters
*
*
* @param kinesisAsyncClient
* the client that will be provided to the RetrievalSpecificConfig constructor
* @param parent

8
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/RetrievalMode.java
View file

@ -19,14 +19,14 @@ import java.util.Arrays;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.commons.lang3.Validate;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.Validate;
@Slf4j
public enum RetrievalMode {
FANOUT(MultiLangDaemonConfiguration::getFanoutConfig), POLLING(
MultiLangDaemonConfiguration::getPollingConfig), DEFAULT(RetrievalMode::decideForDefault);
FANOUT(MultiLangDaemonConfiguration::getFanoutConfig),
POLLING(MultiLangDaemonConfiguration::getPollingConfig),
DEFAULT(RetrievalMode::decideForDefault);
private final Function<MultiLangDaemonConfiguration, RetrievalConfigBuilder> builderFor;

5
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/TypeTag.java
View file

@ -15,14 +15,13 @@
package software.amazon.kinesis.multilang.config;
import lombok.Data;
import java.lang.reflect.Method;
import lombok.Data;
@Data
class TypeTag {
final Class<?> type;
final boolean hasConverter;
final Method builderMethod;
}

5
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/config/credentials/V2CredentialWrapper.java
View file

@ -32,7 +32,10 @@ public class V2CredentialWrapper implements AwsCredentialsProvider {
public AwsCredentials resolveCredentials() {
AWSCredentials current = oldCredentialsProvider.getCredentials();
if (current instanceof AWSSessionCredentials) {
return AwsSessionCredentials.create(current.getAWSAccessKeyId(), current.getAWSSecretKey(), ((AWSSessionCredentials) current).getSessionToken());
return AwsSessionCredentials.create(
current.getAWSAccessKeyId(),
current.getAWSSecretKey(),
((AWSSessionCredentials) current).getSessionToken());
}
return new AwsCredentials() {
@Override

4
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/CheckpointMessage.java
View file

@ -36,6 +36,7 @@ public class CheckpointMessage extends Message {
* The checkpoint this message is about.
*/
private String sequenceNumber;
private Long subSequenceNumber;
/**
@ -45,7 +46,7 @@ public class CheckpointMessage extends Message {
/**
* Convenience constructor.
*
*
* @param sequenceNumber
* The sequence number that this message is about.
* @param subSequenceNumber
@ -61,5 +62,4 @@ public class CheckpointMessage extends Message {
this.setError(throwable.getClass().getSimpleName());
}
}
}

10
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/InitializeMessage.java
View file

@ -33,29 +33,29 @@ public class InitializeMessage extends Message {
* The shard id that this processor is getting initialized for.
*/
private String shardId;
private String sequenceNumber;
private Long subSequenceNumber;
/**
* Default constructor.
*/
public InitializeMessage() {
}
public InitializeMessage() {}
/**
* Convenience constructor.
*
*
* @param initializationInput {@link InitializationInput}
*/
public InitializeMessage(InitializationInput initializationInput) {
this.shardId = initializationInput.shardId();
if (initializationInput.extendedSequenceNumber() != null) {
this.sequenceNumber = initializationInput.extendedSequenceNumber().sequenceNumber();
this.subSequenceNumber = initializationInput.extendedSequenceNumber().subSequenceNumber();
this.subSequenceNumber =
initializationInput.extendedSequenceNumber().subSequenceNumber();
} else {
this.sequenceNumber = null;
this.subSequenceNumber = null;
}
}
}

8
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/JsonFriendlyRecord.java
View file

@ -15,7 +15,6 @@
package software.amazon.kinesis.multilang.messages;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor;
import lombok.EqualsAndHashCode;
import lombok.Getter;
@ -54,10 +53,11 @@ public class JsonFriendlyRecord {
data = new byte[record.data().limit()];
record.data().get(data);
}
Long approximateArrival = record.approximateArrivalTimestamp() == null ? null
Long approximateArrival = record.approximateArrivalTimestamp() == null
? null
: record.approximateArrivalTimestamp().toEpochMilli();
return new JsonFriendlyRecord(data, record.partitionKey(), record.sequenceNumber(),
approximateArrival, record.subSequenceNumber());
return new JsonFriendlyRecord(
data, record.partitionKey(), record.sequenceNumber(), approximateArrival, record.subSequenceNumber());
}
@JsonProperty

1
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/LeaseLostMessage.java
View file

@ -21,5 +21,4 @@ package software.amazon.kinesis.multilang.messages;
public class LeaseLostMessage extends Message {
public static final String ACTION = "leaseLost";
}

25
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/Message.java
View file

@ -23,15 +23,15 @@ import com.fasterxml.jackson.databind.ObjectMapper;
* Abstract class for all messages that are sent to the client's process.
*/
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "action")
@JsonSubTypes({
@Type(value = CheckpointMessage.class, name = CheckpointMessage.ACTION),
@Type(value = InitializeMessage.class, name = InitializeMessage.ACTION),
@Type(value = ProcessRecordsMessage.class, name = ProcessRecordsMessage.ACTION),
@Type(value = ShutdownMessage.class, name = ShutdownMessage.ACTION),
@Type(value = StatusMessage.class, name = StatusMessage.ACTION),
@Type(value = ShutdownRequestedMessage.class, name = ShutdownRequestedMessage.ACTION),
@Type(value = LeaseLostMessage.class, name = LeaseLostMessage.ACTION),
@Type(value = ShardEndedMessage.class, name = ShardEndedMessage.ACTION),
@JsonSubTypes({
@Type(value = CheckpointMessage.class, name = CheckpointMessage.ACTION),
@Type(value = InitializeMessage.class, name = InitializeMessage.ACTION),
@Type(value = ProcessRecordsMessage.class, name = ProcessRecordsMessage.ACTION),
@Type(value = ShutdownMessage.class, name = ShutdownMessage.ACTION),
@Type(value = StatusMessage.class, name = StatusMessage.ACTION),
@Type(value = ShutdownRequestedMessage.class, name = ShutdownRequestedMessage.ACTION),
@Type(value = LeaseLostMessage.class, name = LeaseLostMessage.ACTION),
@Type(value = ShardEndedMessage.class, name = ShardEndedMessage.ACTION),
})
public abstract class Message {
@ -40,11 +40,10 @@ public abstract class Message {
/**
* Default constructor.
*/
public Message() {
}
public Message() {}
/**
*
*
* @param objectMapper An object mapper.
* @return this
*/
@ -54,7 +53,7 @@ public abstract class Message {
}
/**
*
*
* @return A JSON representation of this object.
*/
public String toString() {

6
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/messages/ProcessRecordsMessage.java
View file

@ -37,17 +37,17 @@ public class ProcessRecordsMessage extends Message {
* The records that the client's process needs to handle.
*/
private List<JsonFriendlyRecord> records;
private Long millisBehindLatest;
/**
* Default constructor.
*/
public ProcessRecordsMessage() {
}
public ProcessRecordsMessage() {}
/**
* Convenience constructor.
*
*
* @param processRecordsInput
* the process records input to be sent to the child
*/

3
amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/package-info.java
View file

@ -142,7 +142,6 @@
* Jackson doc for more details)</a> MIME is the basis of most base64 encoding variants including <a
* href="http://tools.ietf.org/html/rfc3548.html">RFC 3548</a> which is the standard used by Python's <a
* href="https://docs.python.org/2/library/base64.html">base64</a> module.
*
*
*/
package software.amazon.kinesis.multilang;

24
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/Matchers.java
View file

@ -14,15 +14,14 @@
*/
package software.amazon.kinesis.multilang;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.nullValue;
import org.hamcrest.Description;
import org.hamcrest.Matcher;
import org.hamcrest.TypeSafeDiagnosingMatcher;
import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber;
import software.amazon.kinesis.lifecycle.events.InitializationInput;
import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.nullValue;
public class Matchers {
@ -58,8 +57,12 @@ public class Matchers {
@Override
public void describeTo(Description description) {
description.appendText("An InitializationInput matching: { shardId: ").appendDescriptionOf(shardIdMatcher)
.appendText(", sequenceNumber: ").appendDescriptionOf(sequenceNumberMatcher).appendText(" }");
description
.appendText("An InitializationInput matching: { shardId: ")
.appendDescriptionOf(shardIdMatcher)
.appendText(", sequenceNumber: ")
.appendDescriptionOf(sequenceNumberMatcher)
.appendText(" }");
}
}
@ -98,10 +101,11 @@ public class Matchers {
@Override
public void describeTo(Description description) {
description.appendText("An ExtendedSequenceNumber matching: { sequenceNumber: ")
.appendDescriptionOf(sequenceNumberMatcher).appendText(", subSequenceNumber: ")
description
.appendText("An ExtendedSequenceNumber matching: { sequenceNumber: ")
.appendDescriptionOf(sequenceNumberMatcher)
.appendText(", subSequenceNumber: ")
.appendDescriptionOf(subSequenceNumberMatcher);
}
}
}

67
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/MessageReaderTest.java
View file

@ -22,22 +22,20 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import software.amazon.kinesis.multilang.MessageReader;
import software.amazon.kinesis.multilang.messages.Message;
import software.amazon.kinesis.multilang.messages.StatusMessage;
import com.fasterxml.jackson.databind.ObjectMapper;
public class MessageReaderTest {
private static final String shardId = "shard-123";
private static final String SHARD_ID = "shard-123";
/*
/**
* This line is based on the definition of the protocol for communication between the KCL record processor and
* the client's process.
*/
@ -45,7 +43,7 @@ public class MessageReaderTest {
return String.format("{\"action\":\"checkpoint\", \"checkpoint\":\"%s\"}", sequenceNumber);
}
/*
/**
* This line is based on the definition of the protocol for communication between the KCL record processor and
* the client's process.
*/
@ -76,18 +74,19 @@ public class MessageReaderTest {
@Test
public void runLoopGoodInputTest() {
String[] sequenceNumbers = new String[] { "123", "456", "789" };
String[] responseFors = new String[] { "initialize", "processRecords", "processRecords", "shutdown" };
String[] sequenceNumbers = new String[] {"123", "456", "789"};
String[] responseFors = new String[] {"initialize", "processRecords", "processRecords", "shutdown"};
InputStream stream = buildInputStreamOfGoodInput(sequenceNumbers, responseFors);
MessageReader reader =
new MessageReader().initialize(stream, shardId, new ObjectMapper(), Executors.newCachedThreadPool());
new MessageReader().initialize(stream, SHARD_ID, new ObjectMapper(), Executors.newCachedThreadPool());
for (String responseFor : responseFors) {
StatusMessage statusMessage = null;
try {
Message message = reader.getNextMessageFromSTDOUT().get();
if (message instanceof StatusMessage) {
Assert.assertEquals("The status message's responseFor field should have been correct", responseFor,
Assert.assertEquals(
"The status message's responseFor field should have been correct",
responseFor,
((StatusMessage) message).getResponseFor());
}
} catch (InterruptedException | ExecutionException e) {
@ -98,19 +97,19 @@ public class MessageReaderTest {
@Test
public void drainInputTest() throws InterruptedException, ExecutionException {
String[] sequenceNumbers = new String[] { "123", "456", "789" };
String[] responseFors = new String[] { "initialize", "processRecords", "processRecords", "shutdown" };
String[] sequenceNumbers = new String[] {"123", "456", "789"};
String[] responseFors = new String[] {"initialize", "processRecords", "processRecords", "shutdown"};
InputStream stream = buildInputStreamOfGoodInput(sequenceNumbers, responseFors);
MessageReader reader =
new MessageReader().initialize(stream, shardId, new ObjectMapper(), Executors.newCachedThreadPool());
new MessageReader().initialize(stream, SHARD_ID, new ObjectMapper(), Executors.newCachedThreadPool());
Future<Boolean> drainFuture = reader.drainSTDOUT();
Boolean drainResult = drainFuture.get();
Assert.assertNotNull(drainResult);
Assert.assertTrue(drainResult);
}
/*
/**
* readValue should fail safely and just continue looping
*/
@Test
@ -118,25 +117,26 @@ public class MessageReaderTest {
BufferedReader bufferReader = Mockito.mock(BufferedReader.class);
try {
Mockito.doAnswer(new Answer() {
private boolean returnedOnce = false;
private boolean returnedOnce = false;
@Override
public Object answer(InvocationOnMock invocation) throws Throwable {
if (returnedOnce) {
return "{\"action\":\"status\",\"responseFor\":\"processRecords\"}";
} else {
returnedOnce = true;
return "{\"action\":\"shutdown\",\"reason\":\"ZOMBIE\"}";
}
}
}).when(bufferReader).readLine();
@Override
public Object answer(InvocationOnMock invocation) throws Throwable {
if (returnedOnce) {
return "{\"action\":\"status\",\"responseFor\":\"processRecords\"}";
} else {
returnedOnce = true;
return "{\"action\":\"shutdown\",\"reason\":\"ZOMBIE\"}";
}
}
})
.when(bufferReader)
.readLine();
} catch (IOException e) {
Assert.fail("There shouldn't be an exception while setting up this mock.");
}
MessageReader reader =
new MessageReader().initialize(bufferReader, shardId, new ObjectMapper(),
Executors.newCachedThreadPool());
MessageReader reader = new MessageReader()
.initialize(bufferReader, SHARD_ID, new ObjectMapper(), Executors.newCachedThreadPool());
try {
reader.getNextMessageFromSTDOUT().get();
@ -150,7 +150,7 @@ public class MessageReaderTest {
public void messageReaderBuilderTest() {
InputStream stream = new ByteArrayInputStream("".getBytes());
MessageReader reader =
new MessageReader().initialize(stream, shardId, new ObjectMapper(), Executors.newCachedThreadPool());
new MessageReader().initialize(stream, SHARD_ID, new ObjectMapper(), Executors.newCachedThreadPool());
Assert.assertNotNull(reader);
}
@ -159,7 +159,7 @@ public class MessageReaderTest {
BufferedReader input = Mockito.mock(BufferedReader.class);
Mockito.doThrow(IOException.class).when(input).readLine();
MessageReader reader =
new MessageReader().initialize(input, shardId, new ObjectMapper(), Executors.newCachedThreadPool());
new MessageReader().initialize(input, SHARD_ID, new ObjectMapper(), Executors.newCachedThreadPool());
Future<Message> readTask = reader.getNextMessageFromSTDOUT();
@ -167,7 +167,8 @@ public class MessageReaderTest {
readTask.get();
Assert.fail("The reading task should have failed due to an IOException.");
} catch (InterruptedException e) {
Assert.fail("The reading task should not have been interrupted. It should have failed due to an IOException.");
Assert.fail(
"The reading task should not have been interrupted. It should have failed due to an IOException.");
} catch (ExecutionException e) {
// Yay!!
}
@ -177,7 +178,7 @@ public class MessageReaderTest {
public void noMoreMessagesTest() throws InterruptedException {
InputStream stream = new ByteArrayInputStream("".getBytes());
MessageReader reader =
new MessageReader().initialize(stream, shardId, new ObjectMapper(), Executors.newCachedThreadPool());
new MessageReader().initialize(stream, SHARD_ID, new ObjectMapper(), Executors.newCachedThreadPool());
Future<Message> future = reader.getNextMessageFromSTDOUT();
try {

68
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/MessageWriterTest.java
View file

@ -23,44 +23,37 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.mockito.Mockito;
import software.amazon.kinesis.lifecycle.events.LeaseLostInput;
import software.amazon.kinesis.lifecycle.events.ShardEndedInput;
import software.amazon.kinesis.multilang.MessageWriter;
import software.amazon.kinesis.multilang.messages.LeaseLostMessage;
import software.amazon.kinesis.multilang.messages.Message;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import software.amazon.kinesis.lifecycle.events.InitializationInput;
import software.amazon.kinesis.lifecycle.events.LeaseLostInput;
import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput;
import software.amazon.kinesis.lifecycle.ShutdownReason;
import software.amazon.kinesis.lifecycle.events.ShardEndedInput;
import software.amazon.kinesis.multilang.messages.Message;
import software.amazon.kinesis.retrieval.KinesisClientRecord;
import static org.mockito.Mockito.verify;
public class MessageWriterTest {
private static final String shardId = "shard-123";
private static final String SHARD_ID = "shard-123";
MessageWriter messageWriter;
OutputStream stream;
@Rule
public final ExpectedException thrown = ExpectedException.none();
// ExecutorService executor;
@Before
public void setup() {
stream = Mockito.mock(OutputStream.class);
messageWriter =
new MessageWriter().initialize(stream, shardId, new ObjectMapper(), Executors.newCachedThreadPool());
new MessageWriter().initialize(stream, SHARD_ID, new ObjectMapper(), Executors.newCachedThreadPool());
}
/*
@ -70,8 +63,7 @@ public class MessageWriterTest {
public void writeCheckpointMessageNoErrorTest() throws IOException, InterruptedException, ExecutionException {
Future<Boolean> future = this.messageWriter.writeCheckpointMessageWithError("1234", 0L, null);
future.get();
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(),
Mockito.anyInt());
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt());
verify(this.stream, Mockito.atLeastOnce()).flush();
}
@ -79,42 +71,43 @@ public class MessageWriterTest {
public void writeCheckpointMessageWithErrorTest() throws IOException, InterruptedException, ExecutionException {
Future<Boolean> future = this.messageWriter.writeCheckpointMessageWithError("1234", 0L, new Throwable());
future.get();
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(),
Mockito.anyInt());
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt());
verify(this.stream, Mockito.atLeastOnce()).flush();
}
@Test
public void writeInitializeMessageTest() throws IOException, InterruptedException, ExecutionException {
Future<Boolean> future = this.messageWriter.writeInitializeMessage(InitializationInput.builder().shardId(shardId).build());
Future<Boolean> future = this.messageWriter.writeInitializeMessage(
InitializationInput.builder().shardId(SHARD_ID).build());
future.get();
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(),
Mockito.anyInt());
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt());
verify(this.stream, Mockito.atLeastOnce()).flush();
}
@Test
public void writeProcessRecordsMessageTest() throws IOException, InterruptedException, ExecutionException {
List<KinesisClientRecord> records = Arrays.asList(
KinesisClientRecord.builder().data(ByteBuffer.wrap("kitten".getBytes())).partitionKey("some cats")
.sequenceNumber("357234807854789057805").build(),
KinesisClientRecord.builder().build()
);
Future<Boolean> future = this.messageWriter.writeProcessRecordsMessage(ProcessRecordsInput.builder().records(records).build());
KinesisClientRecord.builder()
.data(ByteBuffer.wrap("kitten".getBytes()))
.partitionKey("some cats")
.sequenceNumber("357234807854789057805")
.build(),
KinesisClientRecord.builder().build());
Future<Boolean> future = this.messageWriter.writeProcessRecordsMessage(
ProcessRecordsInput.builder().records(records).build());
future.get();
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(),
Mockito.anyInt());
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt());
verify(this.stream, Mockito.atLeastOnce()).flush();
}
@Test
public void writeShutdownMessageTest() throws IOException, InterruptedException, ExecutionException {
Future<Boolean> future = this.messageWriter.writeShardEndedMessage(ShardEndedInput.builder().build());
Future<Boolean> future = this.messageWriter.writeShardEndedMessage(
ShardEndedInput.builder().build());
future.get();
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(),
Mockito.anyInt());
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt());
verify(this.stream, Mockito.atLeastOnce()).flush();
}
@ -123,28 +116,28 @@ public class MessageWriterTest {
Future<Boolean> future = this.messageWriter.writeShutdownRequestedMessage();
future.get();
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(),
Mockito.anyInt());
verify(this.stream, Mockito.atLeastOnce()).write(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt());
verify(this.stream, Mockito.atLeastOnce()).flush();
}
@Test
public void streamIOExceptionTest() throws IOException, InterruptedException, ExecutionException {
Mockito.doThrow(IOException.class).when(stream).flush();
Future<Boolean> initializeTask = this.messageWriter.writeInitializeMessage(InitializationInput.builder().shardId(shardId).build());
Future<Boolean> initializeTask = this.messageWriter.writeInitializeMessage(
InitializationInput.builder().shardId(SHARD_ID).build());
Boolean result = initializeTask.get();
Assert.assertNotNull(result);
Assert.assertFalse(result);
}
@Test
public void objectMapperFails() throws JsonProcessingException, InterruptedException, ExecutionException {
public void objectMapperFails() throws JsonProcessingException {
thrown.expect(RuntimeException.class);
thrown.expectMessage("Encountered I/O error while writing LeaseLostMessage action to subprocess");
ObjectMapper mapper = Mockito.mock(ObjectMapper.class);
Mockito.doThrow(JsonProcessingException.class).when(mapper).writeValueAsString(Mockito.any(Message.class));
messageWriter = new MessageWriter().initialize(stream, shardId, mapper, Executors.newCachedThreadPool());
messageWriter = new MessageWriter().initialize(stream, SHARD_ID, mapper, Executors.newCachedThreadPool());
messageWriter.writeLeaseLossMessage(LeaseLostInput.builder().build());
}
@ -157,7 +150,8 @@ public class MessageWriterTest {
Assert.assertFalse(this.messageWriter.isOpen());
try {
// Any message should fail
this.messageWriter.writeInitializeMessage(InitializationInput.builder().shardId(shardId).build());
this.messageWriter.writeInitializeMessage(
InitializationInput.builder().shardId(SHARD_ID).build());
Assert.fail("MessageWriter should be closed and unable to write.");
} catch (IllegalStateException e) {
// This should happen.

193
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/MultiLangDaemonConfigTest.java
View file

@ -14,83 +14,183 @@
*/
package software.amazon.kinesis.multilang;
import static org.junit.Assert.assertNotNull;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.when;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.Properties;
import org.apache.commons.beanutils.BeanUtilsBean;
import org.apache.commons.beanutils.ConvertUtilsBean;
import org.junit.Before;
import junit.framework.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.runners.MockitoJUnitRunner;
import junit.framework.Assert;
import software.amazon.awssdk.auth.credentials.AwsCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.regions.Region;
import software.amazon.kinesis.multilang.config.KinesisClientLibConfigurator;
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class)
public class MultiLangDaemonConfigTest {
private static String FILENAME = "some.properties";
private static final String FILENAME = "multilang.properties";
private static final String EXE = "TestExe.exe";
private static final String APPLICATION_NAME = MultiLangDaemonConfigTest.class.getSimpleName();
private static final String STREAM_NAME = "fakeStream";
private static final String STREAM_NAME_IN_ARN = "FAKE_STREAM_NAME";
private static final Region REGION = Region.US_EAST_1;
private static final String STREAM_ARN = "arn:aws:kinesis:us-east-2:012345678987:stream/" + STREAM_NAME_IN_ARN;
@Mock
private ClassLoader classLoader;
@Mock
private AwsCredentialsProvider credentialsProvider;
@Mock
private AwsCredentials creds;
@Mock
private KinesisClientLibConfigurator configurator;
@Before
public void setup() {
ConvertUtilsBean convertUtilsBean = new ConvertUtilsBean();
BeanUtilsBean utilsBean = new BeanUtilsBean(convertUtilsBean);
MultiLangDaemonConfiguration multiLangDaemonConfiguration = new MultiLangDaemonConfiguration(utilsBean,
convertUtilsBean);
multiLangDaemonConfiguration.setApplicationName("cool-app");
multiLangDaemonConfiguration.setStreamName("cool-stream");
multiLangDaemonConfiguration.setWorkerIdentifier("cool-worker");
when(credentialsProvider.resolveCredentials()).thenReturn(creds);
when(creds.accessKeyId()).thenReturn("cool-user");
when(configurator.getConfiguration(any(Properties.class))).thenReturn(multiLangDaemonConfiguration);
}
private final KinesisClientLibConfigurator configurator = new KinesisClientLibConfigurator();
private MultiLangDaemonConfig deamonConfig;
@Test
public void constructorTest() throws IOException {
String PROPERTIES = "executableName = randomEXE \n" + "applicationName = testApp \n"
+ "streamName = fakeStream \n" + "AWSCredentialsProvider = DefaultAWSCredentialsProviderChain\n"
+ "processingLanguage = malbolge";
ClassLoader classLoader = Mockito.mock(ClassLoader.class);
/**
* Instantiate a MultiLangDaemonConfig object
* @param streamName
* @param streamArn
* @throws IOException
*/
public void setup(String streamName, String streamArn) throws IOException {
String properties = String.format(
"executableName = %s\n"
+ "applicationName = %s\n"
+ "AWSCredentialsProvider = DefaultAWSCredentialsProviderChain\n"
+ "processingLanguage = malbolge\n"
+ "regionName = %s\n",
EXE, APPLICATION_NAME, "us-east-1");
Mockito.doReturn(new ByteArrayInputStream(PROPERTIES.getBytes())).when(classLoader)
if (streamName != null) {
properties += String.format("streamName = %s\n", streamName);
}
if (streamArn != null) {
properties += String.format("streamArn = %s\n", streamArn);
}
classLoader = Mockito.mock(ClassLoader.class);
Mockito.doReturn(new ByteArrayInputStream(properties.getBytes()))
.when(classLoader)
.getResourceAsStream(FILENAME);
MultiLangDaemonConfig deamonConfig = new MultiLangDaemonConfig(FILENAME, classLoader, configurator);
when(credentialsProvider.resolveCredentials()).thenReturn(creds);
when(creds.accessKeyId()).thenReturn("cool-user");
deamonConfig = new MultiLangDaemonConfig(FILENAME, classLoader, configurator);
}
assertNotNull(deamonConfig.getExecutorService());
assertNotNull(deamonConfig.getMultiLangDaemonConfiguration());
assertNotNull(deamonConfig.getRecordProcessorFactory());
@Test(expected = IllegalArgumentException.class)
public void testConstructorFailsBecauseStreamArnIsInvalid() throws Exception {
setup("", "this_is_not_a_valid_arn");
}
@Test(expected = IllegalArgumentException.class)
public void testConstructorFailsBecauseStreamArnIsInvalid2() throws Exception {
setup("", "arn:aws:kinesis:us-east-2:ACCOUNT_ID:BadFormatting:stream/" + STREAM_NAME_IN_ARN);
}
@Test(expected = IllegalArgumentException.class)
public void testConstructorFailsBecauseStreamNameAndArnAreEmpty() throws Exception {
setup("", "");
}
@Test(expected = NullPointerException.class)
public void testConstructorFailsBecauseStreamNameAndArnAreNull() throws Exception {
setup(null, null);
}
@Test(expected = NullPointerException.class)
public void testConstructorFailsBecauseStreamNameIsNullAndArnIsEmpty() throws Exception {
setup(null, "");
}
@Test(expected = IllegalArgumentException.class)
public void testConstructorFailsBecauseStreamNameIsEmptyAndArnIsNull() throws Exception {
setup("", null);
}
@Test
public void propertyValidation() {
String PROPERTIES_NO_EXECUTABLE_NAME = "applicationName = testApp \n" + "streamName = fakeStream \n"
public void testConstructorUsingStreamName() throws IOException {
setup(STREAM_NAME, null);
assertConfigurationsMatch(STREAM_NAME, null);
}
@Test
public void testConstructorUsingStreamNameAndStreamArnIsEmpty() throws IOException {
setup(STREAM_NAME, "");
assertConfigurationsMatch(STREAM_NAME, "");
}
@Test
public void testConstructorUsingStreamNameAndStreamArnIsWhitespace() throws IOException {
setup(STREAM_NAME, " ");
assertConfigurationsMatch(STREAM_NAME, "");
}
@Test
public void testConstructorUsingStreamArn() throws IOException {
setup(null, STREAM_ARN);
assertConfigurationsMatch(STREAM_NAME_IN_ARN, STREAM_ARN);
}
@Test
public void testConstructorUsingStreamNameAsEmptyAndStreamArn() throws IOException {
setup("", STREAM_ARN);
assertConfigurationsMatch(STREAM_NAME_IN_ARN, STREAM_ARN);
}
@Test
public void testConstructorUsingStreamArnOverStreamName() throws IOException {
setup(STREAM_NAME, STREAM_ARN);
assertConfigurationsMatch(STREAM_NAME_IN_ARN, STREAM_ARN);
}
/**
* Verify the daemonConfig properties are what we expect them to be.
*
* @param expectedStreamName
*/
private void assertConfigurationsMatch(String expectedStreamName, String expectedStreamArn) {
final MultiLangDaemonConfiguration multiLangConfiguration = deamonConfig.getMultiLangDaemonConfiguration();
assertNotNull(deamonConfig.getExecutorService());
assertNotNull(multiLangConfiguration);
assertNotNull(deamonConfig.getRecordProcessorFactory());
assertEquals(EXE, deamonConfig.getRecordProcessorFactory().getCommandArray()[0]);
assertEquals(APPLICATION_NAME, multiLangConfiguration.getApplicationName());
assertEquals(expectedStreamName, multiLangConfiguration.getStreamName());
assertEquals(REGION, multiLangConfiguration.getDynamoDbClient().get("region"));
assertEquals(REGION, multiLangConfiguration.getCloudWatchClient().get("region"));
assertEquals(REGION, multiLangConfiguration.getKinesisClient().get("region"));
assertEquals(expectedStreamArn, multiLangConfiguration.getStreamArn());
}
@Test
public void testPropertyValidation() {
String propertiesNoExecutableName = "applicationName = testApp \n" + "streamName = fakeStream \n"
+ "AWSCredentialsProvider = DefaultAWSCredentialsProviderChain\n" + "processingLanguage = malbolge";
ClassLoader classLoader = Mockito.mock(ClassLoader.class);
Mockito.doReturn(new ByteArrayInputStream(PROPERTIES_NO_EXECUTABLE_NAME.getBytes())).when(classLoader)
Mockito.doReturn(new ByteArrayInputStream(propertiesNoExecutableName.getBytes()))
.when(classLoader)
.getResourceAsStream(FILENAME);
MultiLangDaemonConfig config;
try {
config = new MultiLangDaemonConfig(FILENAME, classLoader, configurator);
new MultiLangDaemonConfig(FILENAME, classLoader, configurator);
Assert.fail("Construction of the config should have failed due to property validation failing.");
} catch (IllegalArgumentException e) {
// Good
@ -99,4 +199,13 @@ public class MultiLangDaemonConfigTest {
}
}
/**
* Test the loading of a "real" properties file. This test should catch
* any issues which might arise if there is a discrepancy between reality
* and mocking.
*/
@Test
public void testActualPropertiesFile() throws Exception {
new MultiLangDaemonConfig(FILENAME);
}
}

59
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/MultiLangDaemonTest.java
View file

@ -14,10 +14,31 @@
*/
package software.amazon.kinesis.multilang;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.classic.joran.JoranConfigurator;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
import org.slf4j.LoggerFactory;
import software.amazon.kinesis.coordinator.Scheduler;
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.isEmptyOrNullString;
import static org.hamcrest.Matchers.nullValue;
import static org.junit.Assert.assertThat;
import static org.mockito.Matchers.anyObject;
@ -28,46 +49,29 @@ import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
import org.slf4j.LoggerFactory;
import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.classic.joran.JoranConfigurator;
import software.amazon.kinesis.coordinator.Scheduler;
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
@RunWith(MockitoJUnitRunner.class)
public class MultiLangDaemonTest {
@Mock
private Scheduler scheduler;
@Mock
private MultiLangDaemonConfig config;
@Mock
private ExecutorService executorService;
@Mock
private Future<Integer> futureInteger;
@Mock
private MultiLangDaemonConfiguration multiLangDaemonConfiguration;
@Mock
private Runtime runtime;
@Rule
public ExpectedException expectedException = ExpectedException.none();
@Rule
public final TemporaryFolder temporaryFolder = new TemporaryFolder();
@ -87,7 +91,7 @@ public class MultiLangDaemonTest {
public void testSuccessfulNoOptionsJCommanderBuild() {
String testPropertiesFile = "/test/properties/file";
MultiLangDaemon.MultiLangDaemonArguments arguments = new MultiLangDaemon.MultiLangDaemonArguments();
daemon.buildJCommanderAndParseArgs(arguments, new String[] { testPropertiesFile });
daemon.buildJCommanderAndParseArgs(arguments, new String[] {testPropertiesFile});
assertThat(arguments.propertiesFile, nullValue());
assertThat(arguments.logConfiguration, nullValue());
@ -99,7 +103,7 @@ public class MultiLangDaemonTest {
public void testSuccessfulOptionsJCommanderBuild() {
String propertiesOption = "/test/properties/file/option";
String propertiesFileArgs = "/test/properties/args";
String[] args = new String[] { "-p", propertiesOption, propertiesFileArgs };
String[] args = new String[] {"-p", propertiesOption, propertiesFileArgs};
MultiLangDaemon.MultiLangDaemonArguments arguments = new MultiLangDaemon.MultiLangDaemonArguments();
daemon.buildJCommanderAndParseArgs(arguments, args);
@ -125,7 +129,8 @@ public class MultiLangDaemonTest {
LoggerContext loggerContext = spy((LoggerContext) LoggerFactory.getILoggerFactory());
JoranConfigurator configurator = spy(new JoranConfigurator());
String logConfiguration = this.getClass().getClassLoader().getResource("logback.xml").getPath();
String logConfiguration =
this.getClass().getClassLoader().getResource("logback.xml").getPath();
daemon.configureLogging(logConfiguration, loggerContext, configurator);
verify(loggerContext).reset();

192
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/MultiLangProtocolTest.java
View file

@ -14,6 +14,42 @@
*/
package software.amazon.kinesis.multilang;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import com.google.common.util.concurrent.SettableFuture;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.runners.MockitoJUnitRunner;
import org.mockito.stubbing.Answer;
import software.amazon.kinesis.exceptions.InvalidStateException;
import software.amazon.kinesis.exceptions.KinesisClientLibDependencyException;
import software.amazon.kinesis.exceptions.ShutdownException;
import software.amazon.kinesis.exceptions.ThrottlingException;
import software.amazon.kinesis.lifecycle.events.InitializationInput;
import software.amazon.kinesis.lifecycle.events.LeaseLostInput;
import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput;
import software.amazon.kinesis.lifecycle.events.ShardEndedInput;
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
import software.amazon.kinesis.multilang.messages.CheckpointMessage;
import software.amazon.kinesis.multilang.messages.LeaseLostMessage;
import software.amazon.kinesis.multilang.messages.Message;
import software.amazon.kinesis.multilang.messages.ProcessRecordsMessage;
import software.amazon.kinesis.multilang.messages.ShardEndedMessage;
import software.amazon.kinesis.multilang.messages.StatusMessage;
import software.amazon.kinesis.processor.RecordProcessorCheckpointer;
import software.amazon.kinesis.retrieval.KinesisClientRecord;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
@ -27,68 +63,35 @@ import static org.mockito.Mockito.timeout;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.runners.MockitoJUnitRunner;
import org.mockito.stubbing.Answer;
import software.amazon.kinesis.exceptions.InvalidStateException;
import software.amazon.kinesis.exceptions.KinesisClientLibDependencyException;
import software.amazon.kinesis.exceptions.ShutdownException;
import software.amazon.kinesis.exceptions.ThrottlingException;
import software.amazon.kinesis.lifecycle.events.LeaseLostInput;
import software.amazon.kinesis.lifecycle.events.ShardEndedInput;
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
import software.amazon.kinesis.multilang.messages.CheckpointMessage;
import software.amazon.kinesis.multilang.messages.LeaseLostMessage;
import software.amazon.kinesis.multilang.messages.Message;
import software.amazon.kinesis.multilang.messages.ProcessRecordsMessage;
import software.amazon.kinesis.multilang.messages.ShardEndedMessage;
import software.amazon.kinesis.multilang.messages.StatusMessage;
import com.google.common.util.concurrent.SettableFuture;
import software.amazon.kinesis.coordinator.KinesisClientLibConfiguration;
import software.amazon.kinesis.lifecycle.events.InitializationInput;
import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput;
import software.amazon.kinesis.lifecycle.ShutdownReason;
import software.amazon.kinesis.processor.RecordProcessorCheckpointer;
import software.amazon.kinesis.retrieval.KinesisClientRecord;
@RunWith(MockitoJUnitRunner.class)
public class MultiLangProtocolTest {
private static final List<KinesisClientRecord> EMPTY_RECORD_LIST = Collections.emptyList();
@Mock
private MultiLangProtocol protocol;
@Mock
private MessageWriter messageWriter;
@Mock
private MessageReader messageReader;
private String shardId;
@Mock
private RecordProcessorCheckpointer checkpointer;
@Mock
private MultiLangDaemonConfiguration configuration;
@Before
public void setup() {
this.shardId = "shard-id-123";
protocol = new MultiLangProtocolForTesting(messageReader, messageWriter,
InitializationInput.builder().shardId(shardId).build(), configuration);
protocol = new MultiLangProtocolForTesting(
messageReader,
messageWriter,
InitializationInput.builder().shardId(shardId).build(),
configuration);
when(configuration.getTimeoutInSeconds()).thenReturn(null);
}
@ -106,38 +109,42 @@ public class MultiLangProtocolTest {
}
@Test
public void initializeTest() throws InterruptedException, ExecutionException {
when(messageWriter
.writeInitializeMessage(argThat(Matchers.withInit(InitializationInput.builder()
.shardId(shardId).build())))).thenReturn(buildFuture(true));
when(messageReader.getNextMessageFromSTDOUT()).thenReturn(buildFuture(
new StatusMessage("initialize"), Message.class));
public void testInitialize() {
when(messageWriter.writeInitializeMessage(argThat(Matchers.withInit(
InitializationInput.builder().shardId(shardId).build()))))
.thenReturn(buildFuture(true));
when(messageReader.getNextMessageFromSTDOUT())
.thenReturn(buildFuture(new StatusMessage("initialize"), Message.class));
assertThat(protocol.initialize(), equalTo(true));
}
@Test
public void processRecordsTest() throws InterruptedException, ExecutionException {
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class))).thenReturn(buildFuture(true));
when(messageReader.getNextMessageFromSTDOUT()).thenReturn(buildFuture(
new StatusMessage("processRecords"), Message.class));
public void testProcessRecords() {
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class)))
.thenReturn(buildFuture(true));
when(messageReader.getNextMessageFromSTDOUT())
.thenReturn(buildFuture(new StatusMessage("processRecords"), Message.class));
assertThat(protocol.processRecords(ProcessRecordsInput.builder().records(EMPTY_RECORD_LIST).build()),
assertThat(
protocol.processRecords(
ProcessRecordsInput.builder().records(EMPTY_RECORD_LIST).build()),
equalTo(true));
}
@Test
public void leaseLostTest() {
when(messageWriter.writeLeaseLossMessage(any(LeaseLostInput.class))).thenReturn(buildFuture(true));
when(messageReader.getNextMessageFromSTDOUT()).thenReturn(buildFuture(new StatusMessage(LeaseLostMessage.ACTION), Message.class));
when(messageReader.getNextMessageFromSTDOUT())
.thenReturn(buildFuture(new StatusMessage(LeaseLostMessage.ACTION), Message.class));
assertThat(protocol.leaseLost(LeaseLostInput.builder().build()), equalTo(true));
}
@Test
public void shardEndedTest() {
when(messageWriter.writeShardEndedMessage(any(ShardEndedInput.class))).thenReturn(buildFuture(true));
when(messageReader.getNextMessageFromSTDOUT()).thenReturn(buildFuture(new StatusMessage(ShardEndedMessage.ACTION)));
when(messageReader.getNextMessageFromSTDOUT())
.thenReturn(buildFuture(new StatusMessage(ShardEndedMessage.ACTION)));
assertThat(protocol.shardEnded(ShardEndedInput.builder().build()), equalTo(true));
}
@ -145,12 +152,12 @@ public class MultiLangProtocolTest {
@Test
public void shutdownRequestedTest() {
when(messageWriter.writeShutdownRequestedMessage()).thenReturn(buildFuture(true));
when(messageReader.getNextMessageFromSTDOUT()).thenReturn(buildFuture(
new StatusMessage("shutdownRequested"), Message.class));
Mockito.doReturn(buildFuture(true)).when(messageWriter)
.writeShutdownRequestedMessage();
when(messageReader.getNextMessageFromSTDOUT())
.thenReturn(buildFuture(new StatusMessage("shutdownRequested"), Message.class));
Mockito.doReturn(buildFuture(true)).when(messageWriter).writeShutdownRequestedMessage();
Mockito.doReturn(buildFuture(new StatusMessage("shutdownRequested")))
.when(messageReader).getNextMessageFromSTDOUT();
.when(messageReader)
.getNextMessageFromSTDOUT();
assertThat(protocol.shutdownRequested(null), equalTo(true));
}
@ -172,16 +179,17 @@ public class MultiLangProtocolTest {
}
return buildFuture(message);
}
}.init(messages);
}
@Test
public void processRecordsWithCheckpointsTest() throws InterruptedException, ExecutionException,
KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
public void testProcessRecordsWithCheckpoints()
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class))).thenReturn(buildFuture(true));
when(messageWriter.writeCheckpointMessageWithError(anyString(), anyLong(), any(Throwable.class))).thenReturn(buildFuture(true));
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class)))
.thenReturn(buildFuture(true));
when(messageWriter.writeCheckpointMessageWithError(anyString(), anyLong(), any(Throwable.class)))
.thenReturn(buildFuture(true));
when(messageReader.getNextMessageFromSTDOUT()).thenAnswer(buildMessageAnswers(new ArrayList<Message>() {
{
this.add(new CheckpointMessage("123", 0L, null));
@ -196,8 +204,10 @@ public class MultiLangProtocolTest {
}
}));
boolean result = protocol.processRecords(ProcessRecordsInput.builder().records(EMPTY_RECORD_LIST)
.checkpointer(checkpointer).build());
boolean result = protocol.processRecords(ProcessRecordsInput.builder()
.records(EMPTY_RECORD_LIST)
.checkpointer(checkpointer)
.build());
assertThat(result, equalTo(true));
@ -206,42 +216,53 @@ public class MultiLangProtocolTest {
}
@Test
public void processRecordsWithABadCheckpointTest() throws InterruptedException, ExecutionException {
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class))).thenReturn(buildFuture(true));
when(messageWriter.writeCheckpointMessageWithError(anyString(), anyLong(), any(Throwable.class))).thenReturn(buildFuture(false));
public void testProcessRecordsWithABadCheckpoint() {
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class)))
.thenReturn(buildFuture(true));
when(messageWriter.writeCheckpointMessageWithError(anyString(), anyLong(), any(Throwable.class)))
.thenReturn(buildFuture(false));
when(messageReader.getNextMessageFromSTDOUT()).thenAnswer(buildMessageAnswers(new ArrayList<Message>() {
{
this.add(new CheckpointMessage("456", 0L, null));
this.add(new StatusMessage("processRecords"));
}
}));
assertThat(protocol.processRecords(ProcessRecordsInput.builder().records(EMPTY_RECORD_LIST)
.checkpointer(checkpointer).build()), equalTo(false));
assertThat(
protocol.processRecords(ProcessRecordsInput.builder()
.records(EMPTY_RECORD_LIST)
.checkpointer(checkpointer)
.build()),
equalTo(false));
}
@Test(expected = NullPointerException.class)
public void waitForStatusMessageTimeoutTest() throws InterruptedException, TimeoutException, ExecutionException {
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class))).thenReturn(buildFuture(true));
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class)))
.thenReturn(buildFuture(true));
Future<Message> future = Mockito.mock(Future.class);
when(messageReader.getNextMessageFromSTDOUT()).thenReturn(future);
when(configuration.getTimeoutInSeconds()).thenReturn(5);
when(future.get(anyInt(), eq(TimeUnit.SECONDS))).thenThrow(TimeoutException.class);
protocol = new MultiLangProtocolForTesting(messageReader,
protocol = new MultiLangProtocolForTesting(
messageReader,
messageWriter,
InitializationInput.builder().shardId(shardId).build(),
configuration);
protocol.processRecords(ProcessRecordsInput.builder().records(EMPTY_RECORD_LIST).build());
protocol.processRecords(
ProcessRecordsInput.builder().records(EMPTY_RECORD_LIST).build());
}
@Test
public void waitForStatusMessageSuccessTest() {
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class))).thenReturn(buildFuture(true));
when(messageReader.getNextMessageFromSTDOUT()).thenReturn(buildFuture(
new StatusMessage("processRecords"), Message.class));
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class)))
.thenReturn(buildFuture(true));
when(messageReader.getNextMessageFromSTDOUT())
.thenReturn(buildFuture(new StatusMessage("processRecords"), Message.class));
when(configuration.getTimeoutInSeconds()).thenReturn(5);
assertTrue(protocol.processRecords(ProcessRecordsInput.builder().records(EMPTY_RECORD_LIST).build()));
assertTrue(protocol.processRecords(
ProcessRecordsInput.builder().records(EMPTY_RECORD_LIST).build()));
}
private class MultiLangProtocolForTesting extends MultiLangProtocol {
@ -253,10 +274,11 @@ public class MultiLangProtocolTest {
* @param initializationInput
* @param configuration
*/
MultiLangProtocolForTesting(final MessageReader messageReader,
final MessageWriter messageWriter,
final InitializationInput initializationInput,
final MultiLangDaemonConfiguration configuration) {
MultiLangProtocolForTesting(
final MessageReader messageReader,
final MessageWriter messageWriter,
final InitializationInput initializationInput,
final MultiLangDaemonConfiguration configuration) {
super(messageReader, messageWriter, initializationInput, configuration);
}

110
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/NestedPropertyKeyTest.java Normal file
View file

@ -0,0 +1,110 @@
/*
* Copyright 2023 Amazon.com, Inc. or its affiliates.
* Licensed under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package software.amazon.kinesis.multilang;
import com.amazonaws.regions.Regions;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyZeroInteractions;
import static software.amazon.kinesis.multilang.NestedPropertyKey.ENDPOINT;
import static software.amazon.kinesis.multilang.NestedPropertyKey.ENDPOINT_REGION;
import static software.amazon.kinesis.multilang.NestedPropertyKey.EXTERNAL_ID;
import static software.amazon.kinesis.multilang.NestedPropertyKey.parse;
@RunWith(MockitoJUnitRunner.class)
public class NestedPropertyKeyTest {
@Mock
private NestedPropertyProcessor mockProcessor;
@Test
public void testExternalId() {
final String expectedId = "eid";
parse(mockProcessor, createKey(EXTERNAL_ID, expectedId));
verify(mockProcessor).acceptExternalId(expectedId);
}
@Test
public void testEndpoint() {
final String expectedEndpoint = "https://sts.us-east-1.amazonaws.com";
final String expectedRegion = "us-east-1";
final String param = createKey(ENDPOINT, expectedEndpoint + "^" + expectedRegion);
parse(mockProcessor, param);
verify(mockProcessor).acceptEndpoint(expectedEndpoint, expectedRegion);
}
@Test(expected = IllegalArgumentException.class)
public void testInvalidEndpoint() {
parse(mockProcessor, createKey(ENDPOINT, "value-sans-caret-delimiter"));
}
@Test(expected = IllegalArgumentException.class)
public void testInvalidEndpointDoubleCaret() {
parse(mockProcessor, createKey(ENDPOINT, "https://sts.us-east-1.amazonaws.com^us-east-1^borkbork"));
}
@Test
public void testEndpointRegion() {
final Regions expectedRegion = Regions.GovCloud;
parse(mockProcessor, createKey(ENDPOINT_REGION, expectedRegion.getName()));
verify(mockProcessor).acceptEndpointRegion(expectedRegion);
}
@Test(expected = IllegalArgumentException.class)
public void testInvalidEndpointRegion() {
parse(mockProcessor, createKey(ENDPOINT_REGION, "snuffleupagus"));
}
/**
* Test that the literal nested key (i.e., {@code key=} in {@code some_val|key=nested_val})
* does not change. Any change to an existing literal key is not backwards-compatible.
*/
@Test
public void testKeysExplicitly() {
// Adding a new enum will deliberately cause this assert to fail, and
// therefore raise awareness for this explicit test. Add-and-remove may
// keep the number unchanged yet will also break (by removing an enum).
assertEquals(3, NestedPropertyKey.values().length);
assertEquals("endpoint", ENDPOINT.getNestedKey());
assertEquals("endpointRegion", ENDPOINT_REGION.getNestedKey());
assertEquals("externalId", EXTERNAL_ID.getNestedKey());
}
@Test
public void testNonmatchingParameters() {
final String[] params = new String[] {
null,
"",
"hello world", // no nested key
"foo=bar", // nested key, but is not a recognized key
createKey(EXTERNAL_ID, "eid") + "=extra", // valid key made invalid by second '='
};
parse(mockProcessor, params);
verifyZeroInteractions(mockProcessor);
}
private static String createKey(final NestedPropertyKey key, final String value) {
return key.getNestedKey() + "=" + value;
}
}

13
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/ReadSTDERRTaskTest.java
View file

@ -27,12 +27,10 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
import software.amazon.kinesis.multilang.DrainChildSTDERRTask;
import software.amazon.kinesis.multilang.LineReaderTask;
public class ReadSTDERRTaskTest {
private static final String shardId = "shard-123";
private static final String SHARD_ID = "shard-123";
private BufferedReader mockBufferReader;
@Before
@ -45,7 +43,7 @@ public class ReadSTDERRTaskTest {
String errorMessages = "OMG\nThis is test message\n blah blah blah \n";
InputStream stream = new ByteArrayInputStream(errorMessages.getBytes());
LineReaderTask<Boolean> reader = new DrainChildSTDERRTask().initialize(stream, shardId, "");
LineReaderTask<Boolean> reader = new DrainChildSTDERRTask().initialize(stream, SHARD_ID, "");
Assert.assertNotNull(reader);
}
@ -54,7 +52,7 @@ public class ReadSTDERRTaskTest {
String errorMessages = "OMG\nThis is test message\n blah blah blah \n";
BufferedReader bufferReader =
new BufferedReader(new InputStreamReader(new ByteArrayInputStream(errorMessages.getBytes())));
LineReaderTask<Boolean> errorReader = new DrainChildSTDERRTask().initialize(bufferReader, shardId, "");
LineReaderTask<Boolean> errorReader = new DrainChildSTDERRTask().initialize(bufferReader, SHARD_ID, "");
Assert.assertNotNull(errorReader);
Boolean result = errorReader.call();
@ -67,14 +65,15 @@ public class ReadSTDERRTaskTest {
} catch (IOException e) {
Assert.fail("Not supposed to get an exception when we're just building our mock.");
}
LineReaderTask<Boolean> errorReader = new DrainChildSTDERRTask().initialize(mockBufferReader, shardId, "");
LineReaderTask<Boolean> errorReader = new DrainChildSTDERRTask().initialize(mockBufferReader, SHARD_ID, "");
Assert.assertNotNull(errorReader);
Future<Boolean> result = Executors.newCachedThreadPool().submit(errorReader);
Boolean finishedCleanly = null;
try {
finishedCleanly = result.get();
} catch (InterruptedException | ExecutionException e) {
Assert.fail("Should have been able to get a result. The error should be handled during the call and result in false.");
Assert.fail(
"Should have been able to get a result. The error should be handled during the call and result in false.");
}
Assert.assertFalse("Reading a line should have thrown an exception", finishedCleanly);
}

15
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/StreamingShardRecordProcessorFactoryTest.java
View file

@ -14,17 +14,13 @@
*/
package software.amazon.kinesis.multilang;
import software.amazon.kinesis.coordinator.KinesisClientLibConfiguration;
import org.junit.Assert;
import org.junit.Test;
import software.amazon.kinesis.multilang.MultiLangRecordProcessorFactory;
import software.amazon.kinesis.multilang.MultiLangShardRecordProcessor;
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
import software.amazon.kinesis.processor.ShardRecordProcessor;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
import software.amazon.kinesis.multilang.config.MultiLangDaemonConfiguration;
import software.amazon.kinesis.processor.ShardRecordProcessor;
@RunWith(MockitoJUnitRunner.class)
public class StreamingShardRecordProcessorFactoryTest {
@ -34,10 +30,13 @@ public class StreamingShardRecordProcessorFactoryTest {
@Test
public void createProcessorTest() {
MultiLangRecordProcessorFactory factory = new MultiLangRecordProcessorFactory("somecommand", null, configuration);
MultiLangRecordProcessorFactory factory =
new MultiLangRecordProcessorFactory("somecommand", null, configuration);
ShardRecordProcessor processor = factory.shardRecordProcessor();
Assert.assertEquals("Should have constructed a StreamingRecordProcessor", MultiLangShardRecordProcessor.class,
Assert.assertEquals(
"Should have constructed a StreamingRecordProcessor",
MultiLangShardRecordProcessor.class,
processor.getClass());
}
}

130
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/StreamingShardRecordProcessorTest.java
View file

@ -14,16 +14,6 @@
*/
package software.amazon.kinesis.multilang;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Matchers.anyString;
import static org.mockito.Matchers.argThat;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Collections;
@ -33,6 +23,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@ -42,13 +33,8 @@ import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.runners.MockitoJUnitRunner;
import org.mockito.stubbing.Answer;
import com.fasterxml.jackson.databind.ObjectMapper;
import software.amazon.awssdk.services.kinesis.model.Record;
import software.amazon.kinesis.exceptions.InvalidStateException;
import software.amazon.kinesis.exceptions.KinesisClientLibDependencyException;
import software.amazon.kinesis.exceptions.ShutdownException;
import software.amazon.kinesis.exceptions.ThrottlingException;
import software.amazon.kinesis.lifecycle.events.InitializationInput;
import software.amazon.kinesis.lifecycle.events.LeaseLostInput;
@ -64,92 +50,98 @@ import software.amazon.kinesis.processor.PreparedCheckpointer;
import software.amazon.kinesis.processor.RecordProcessorCheckpointer;
import software.amazon.kinesis.retrieval.KinesisClientRecord;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Matchers.anyString;
import static org.mockito.Matchers.argThat;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class)
public class StreamingShardRecordProcessorTest {
private static final String shardId = "shard-123";
private static final String SHARD_ID = "shard-123";
private int systemExitCount = 0;
@Mock
private Future<Message> messageFuture;
@Mock
private Future<Boolean> trueFuture;
private RecordProcessorCheckpointer unimplementedCheckpointer = new RecordProcessorCheckpointer() {
@Override
public void checkpoint() throws KinesisClientLibDependencyException, InvalidStateException,
ThrottlingException, ShutdownException {
public void checkpoint() throws KinesisClientLibDependencyException, ThrottlingException {
throw new UnsupportedOperationException();
}
@Override
public void checkpoint(String sequenceNumber) throws KinesisClientLibDependencyException,
InvalidStateException, ThrottlingException, ShutdownException, IllegalArgumentException {
public void checkpoint(String sequenceNumber)
throws KinesisClientLibDependencyException, ThrottlingException, IllegalArgumentException {
throw new UnsupportedOperationException();
}
@Override
public void checkpoint(Record record)
throws KinesisClientLibDependencyException,
InvalidStateException, ThrottlingException, ShutdownException {
public void checkpoint(Record record) throws KinesisClientLibDependencyException, ThrottlingException {
throw new UnsupportedOperationException();
}
@Override
public void checkpoint(String sequenceNumber, long subSequenceNumber)
throws KinesisClientLibDependencyException,
InvalidStateException, ThrottlingException, ShutdownException,
IllegalArgumentException {
throws KinesisClientLibDependencyException, ThrottlingException, IllegalArgumentException {
throw new UnsupportedOperationException();
}
@Override
public PreparedCheckpointer prepareCheckpoint()
throws KinesisClientLibDependencyException,
InvalidStateException, ThrottlingException, ShutdownException {
throws KinesisClientLibDependencyException, ThrottlingException {
throw new UnsupportedOperationException();
}
@Override
public PreparedCheckpointer prepareCheckpoint(byte[] applicationState) throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
public PreparedCheckpointer prepareCheckpoint(byte[] applicationState)
throws KinesisClientLibDependencyException, ThrottlingException {
throw new UnsupportedOperationException();
}
@Override
public PreparedCheckpointer prepareCheckpoint(Record record)
throws KinesisClientLibDependencyException,
InvalidStateException, ThrottlingException, ShutdownException {
throws KinesisClientLibDependencyException, ThrottlingException {
throw new UnsupportedOperationException();
}
@Override
public PreparedCheckpointer prepareCheckpoint(Record record, byte[] applicationState) throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
public PreparedCheckpointer prepareCheckpoint(Record record, byte[] applicationState)
throws KinesisClientLibDependencyException, ThrottlingException {
throw new UnsupportedOperationException();
}
@Override
public PreparedCheckpointer prepareCheckpoint(String sequenceNumber)
throws KinesisClientLibDependencyException,
InvalidStateException, ThrottlingException, ShutdownException, IllegalArgumentException {
throws KinesisClientLibDependencyException, ThrottlingException, IllegalArgumentException {
throw new UnsupportedOperationException();
}
@Override
public PreparedCheckpointer prepareCheckpoint(String sequenceNumber, byte[] applicationState) throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException, IllegalArgumentException {
public PreparedCheckpointer prepareCheckpoint(String sequenceNumber, byte[] applicationState)
throws KinesisClientLibDependencyException, ThrottlingException, IllegalArgumentException {
return null;
}
@Override
public PreparedCheckpointer prepareCheckpoint(String sequenceNumber, long subSequenceNumber)
throws KinesisClientLibDependencyException,
InvalidStateException, ThrottlingException, ShutdownException, IllegalArgumentException {
throws KinesisClientLibDependencyException, ThrottlingException, IllegalArgumentException {
throw new UnsupportedOperationException();
}
@Override
public PreparedCheckpointer prepareCheckpoint(String sequenceNumber, long subSequenceNumber, byte[] applicationState) throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException, IllegalArgumentException {
public PreparedCheckpointer prepareCheckpoint(
String sequenceNumber, long subSequenceNumber, byte[] applicationState)
throws KinesisClientLibDependencyException, ThrottlingException, IllegalArgumentException {
throw new UnsupportedOperationException();
}
@ -171,7 +163,7 @@ public class StreamingShardRecordProcessorTest {
private MultiLangDaemonConfiguration configuration;
@Before
public void prepare() throws IOException, InterruptedException, ExecutionException {
public void prepare() throws InterruptedException, ExecutionException {
// Fake command
systemExitCount = 0;
@ -185,8 +177,14 @@ public class StreamingShardRecordProcessorTest {
when(configuration.getTimeoutInSeconds()).thenReturn(null);
recordProcessor =
new MultiLangShardRecordProcessor(new ProcessBuilder(), executor, new ObjectMapper(), messageWriter,
messageReader, errorReader, configuration) {
new MultiLangShardRecordProcessor(
new ProcessBuilder(),
executor,
new ObjectMapper(),
messageWriter,
messageReader,
errorReader,
configuration) {
// Just don't do anything when we exit.
void exit() {
@ -210,9 +208,12 @@ public class StreamingShardRecordProcessorTest {
Mockito.doReturn(Mockito.mock(Future.class)).when(messageReader).drainSTDOUT();
Mockito.doReturn(true).when(trueFuture).get();
when(messageWriter.writeInitializeMessage(any(InitializationInput.class))).thenReturn(trueFuture);
when(messageWriter.writeCheckpointMessageWithError(anyString(), anyLong(), any(Throwable.class))).thenReturn(trueFuture);
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class))).thenReturn(trueFuture);
when(messageWriter.writeInitializeMessage(any(InitializationInput.class)))
.thenReturn(trueFuture);
when(messageWriter.writeCheckpointMessageWithError(anyString(), anyLong(), any(Throwable.class)))
.thenReturn(trueFuture);
when(messageWriter.writeProcessRecordsMessage(any(ProcessRecordsInput.class)))
.thenReturn(trueFuture);
when(messageWriter.writeLeaseLossMessage(any(LeaseLostInput.class))).thenReturn(trueFuture);
}
@ -230,22 +231,29 @@ public class StreamingShardRecordProcessorTest {
List<KinesisClientRecord> testRecords = Collections.emptyList();
recordProcessor.initialize(InitializationInput.builder().shardId(shardId).build());
recordProcessor.processRecords(ProcessRecordsInput.builder().records(testRecords)
.checkpointer(unimplementedCheckpointer).build());
recordProcessor.processRecords(ProcessRecordsInput.builder().records(testRecords)
.checkpointer(unimplementedCheckpointer).build());
recordProcessor.initialize(
InitializationInput.builder().shardId(SHARD_ID).build());
recordProcessor.processRecords(ProcessRecordsInput.builder()
.records(testRecords)
.checkpointer(unimplementedCheckpointer)
.build());
recordProcessor.processRecords(ProcessRecordsInput.builder()
.records(testRecords)
.checkpointer(unimplementedCheckpointer)
.build());
recordProcessor.leaseLost(LeaseLostInput.builder().build());
}
@Test
public void processorPhasesTest() throws InterruptedException, ExecutionException {
Answer<StatusMessage> answer = new Answer<StatusMessage>() {
StatusMessage[] answers = new StatusMessage[] { new StatusMessage(InitializeMessage.ACTION),
new StatusMessage(ProcessRecordsMessage.ACTION), new StatusMessage(ProcessRecordsMessage.ACTION),
new StatusMessage(ShutdownMessage.ACTION) };
StatusMessage[] answers = new StatusMessage[] {
new StatusMessage(InitializeMessage.ACTION),
new StatusMessage(ProcessRecordsMessage.ACTION),
new StatusMessage(ProcessRecordsMessage.ACTION),
new StatusMessage(ShutdownMessage.ACTION)
};
int callCount = 0;
@ -263,7 +271,7 @@ public class StreamingShardRecordProcessorTest {
verify(messageWriter)
.writeInitializeMessage(argThat(Matchers.withInit(
InitializationInput.builder().shardId(shardId).build())));
InitializationInput.builder().shardId(SHARD_ID).build())));
verify(messageWriter, times(2)).writeProcessRecordsMessage(any(ProcessRecordsInput.class));
verify(messageWriter).writeLeaseLossMessage(any(LeaseLostInput.class));
}
@ -276,9 +284,12 @@ public class StreamingShardRecordProcessorTest {
* This bad message will cause shutdown to not attempt to send a message. i.e. avoid encountering an
* exception.
*/
StatusMessage[] answers = new StatusMessage[] { new StatusMessage("Bad"),
new StatusMessage(ProcessRecordsMessage.ACTION), new StatusMessage(ProcessRecordsMessage.ACTION),
new StatusMessage(ShutdownMessage.ACTION) };
StatusMessage[] answers = new StatusMessage[] {
new StatusMessage("Bad"),
new StatusMessage(ProcessRecordsMessage.ACTION),
new StatusMessage(ProcessRecordsMessage.ACTION),
new StatusMessage(ShutdownMessage.ACTION)
};
int callCount = 0;
@ -294,8 +305,9 @@ public class StreamingShardRecordProcessorTest {
phases(answer);
verify(messageWriter).writeInitializeMessage(argThat(Matchers.withInit(InitializationInput.builder()
.shardId(shardId).build())));
verify(messageWriter)
.writeInitializeMessage(argThat(Matchers.withInit(
InitializationInput.builder().shardId(SHARD_ID).build())));
verify(messageWriter, times(2)).writeProcessRecordsMessage(any(ProcessRecordsInput.class));
verify(messageWriter, never()).writeLeaseLossMessage(any(LeaseLostInput.class));
Assert.assertEquals(1, systemExitCount);

70
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/auth/KclSTSAssumeRoleSessionCredentialsProviderTest.java Normal file
View file

@ -0,0 +1,70 @@
/*
* Copyright 2023 Amazon.com, Inc. or its affiliates.
* Licensed under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package software.amazon.kinesis.multilang.auth;
import java.util.Arrays;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
public class KclSTSAssumeRoleSessionCredentialsProviderTest {
private static final String ARN = "arn";
private static final String SESSION_NAME = "sessionName";
/**
* Test that the constructor doesn't throw an out-of-bounds exception if
* there are no parameters beyond the required ARN and session name.
*/
@Test
public void testConstructorWithoutOptionalParams() {
new KclSTSAssumeRoleSessionCredentialsProvider(new String[] {ARN, SESSION_NAME});
}
@Test
public void testAcceptEndpoint() {
// discovered exception during e2e testing; therefore, this test is
// to simply verify the constructed STS client doesn't go *boom*
final KclSTSAssumeRoleSessionCredentialsProvider provider =
new KclSTSAssumeRoleSessionCredentialsProvider(ARN, SESSION_NAME);
provider.acceptEndpoint("endpoint", "us-east-1");
}
@Test
public void testVarArgs() {
for (final String[] varargs : Arrays.asList(
new String[] {ARN, SESSION_NAME, "externalId=eid", "foo"},
new String[] {ARN, SESSION_NAME, "foo", "externalId=eid"})) {
final VarArgsSpy provider = new VarArgsSpy(varargs);
assertEquals("eid", provider.externalId);
}
}
private static class VarArgsSpy extends KclSTSAssumeRoleSessionCredentialsProvider {
private String externalId;
public VarArgsSpy(String[] args) {
super(args);
}
@Override
public void acceptExternalId(final String externalId) {
this.externalId = externalId;
super.acceptExternalId(externalId);
}
}
}

90
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/AWSCredentialsProviderPropertyValueDecoderTest.java
View file

@ -14,37 +14,33 @@
*/
package software.amazon.kinesis.multilang.config;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import java.util.Arrays;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.AWSCredentialsProviderChain;
import com.amazonaws.auth.BasicAWSCredentials;
import lombok.ToString;
import org.hamcrest.Description;
import org.hamcrest.Matcher;
import org.hamcrest.TypeSafeDiagnosingMatcher;
import org.junit.Test;
import software.amazon.kinesis.multilang.auth.KclSTSAssumeRoleSessionCredentialsProvider;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.AWSCredentialsProviderChain;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThat;
public class AWSCredentialsProviderPropertyValueDecoderTest {
private static final String TEST_ACCESS_KEY_ID = "123";
private static final String TEST_SECRET_KEY = "456";
private String credentialName1 = "software.amazon.kinesis.multilang.config.AWSCredentialsProviderPropertyValueDecoderTest$AlwaysSucceedCredentialsProvider";
private String credentialName2 = "software.amazon.kinesis.multilang.config.AWSCredentialsProviderPropertyValueDecoderTest$ConstructorCredentialsProvider";
private AWSCredentialsProviderPropertyValueDecoder decoder = new AWSCredentialsProviderPropertyValueDecoder();
private final String credentialName1 = AlwaysSucceedCredentialsProvider.class.getName();
private final String credentialName2 = ConstructorCredentialsProvider.class.getName();
private final AWSCredentialsProviderPropertyValueDecoder decoder = new AWSCredentialsProviderPropertyValueDecoder();
@ToString
private static class AWSCredentialsMatcher extends TypeSafeDiagnosingMatcher<AWSCredentialsProvider> {
@ -59,10 +55,6 @@ public class AWSCredentialsProviderPropertyValueDecoderTest {
this.classMatcher = instanceOf(AWSCredentialsProviderChain.class);
}
private AWSCredentialsMatcher(AWSCredentials expected) {
this(expected.getAWSAccessKeyId(), expected.getAWSSecretKey());
}
@Override
protected boolean matchesSafely(AWSCredentialsProvider item, Description mismatchDescription) {
AWSCredentials actual = item.getCredentials();
@ -86,10 +78,10 @@ public class AWSCredentialsProviderPropertyValueDecoderTest {
@Override
public void describeTo(Description description) {
description.appendText("An AWSCredentialsProvider that provides an AWSCredential matching: ")
description
.appendText("An AWSCredentialsProvider that provides an AWSCredential matching: ")
.appendList("(", ", ", ")", Arrays.asList(classMatcher, akidMatcher, secretMatcher));
}
}
private static AWSCredentialsMatcher hasCredentials(String akid, String secret) {
@ -120,6 +112,33 @@ public class AWSCredentialsProviderPropertyValueDecoderTest {
assertThat(provider, hasCredentials("arg1", "arg2"));
}
/**
* Test that providers in the multi-lang auth package can be resolved and instantiated.
*/
@Test
public void testKclAuthProvider() {
for (final String className : Arrays.asList(
KclSTSAssumeRoleSessionCredentialsProvider.class.getName(), // fully-qualified name
KclSTSAssumeRoleSessionCredentialsProvider.class.getSimpleName() // name-only; needs prefix
)) {
final AWSCredentialsProvider provider = decoder.decodeValue(className + "|arn|sessionName");
assertNotNull(className, provider);
}
}
/**
* Test that a provider can be instantiated by its varargs constructor.
*/
@Test
public void testVarArgAuthProvider() {
final String[] args = new String[] {"arg1", "arg2", "arg3"};
final String className = VarArgCredentialsProvider.class.getName();
final String encodedValue = className + "|" + String.join("|", args);
final AWSCredentialsProvider provider = decoder.decodeValue(encodedValue);
assertEquals(Arrays.toString(args), provider.getCredentials().getAWSAccessKeyId());
}
/**
* This credentials provider will always succeed
*/
@ -131,9 +150,7 @@ public class AWSCredentialsProviderPropertyValueDecoderTest {
}
@Override
public void refresh() {
}
public void refresh() {}
}
/**
@ -144,9 +161,9 @@ public class AWSCredentialsProviderPropertyValueDecoderTest {
private String arg1;
private String arg2;
@SuppressWarnings("unused")
public ConstructorCredentialsProvider(String arg1) {
this.arg1 = arg1;
this.arg2 = "blank";
this(arg1, "blank");
}
public ConstructorCredentialsProvider(String arg1, String arg2) {
@ -160,8 +177,25 @@ public class AWSCredentialsProviderPropertyValueDecoderTest {
}
@Override
public void refresh() {
public void refresh() {}
}
private static class VarArgCredentialsProvider implements AWSCredentialsProvider {
private final String[] args;
public VarArgCredentialsProvider(final String[] args) {
this.args = args;
}
@Override
public AWSCredentials getCredentials() {
// KISS solution to surface the constructor args
final String flattenedArgs = Arrays.toString(args);
return new BasicAWSCredentials(flattenedArgs, flattenedArgs);
}
@Override
public void refresh() {}
}
}

218
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/BuilderDynaBeanTest.java
View file

@ -15,16 +15,14 @@
package software.amazon.kinesis.multilang.config;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.CoreMatchers.nullValue;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.fail;
import java.util.function.Consumer;
import java.util.function.Supplier;
import lombok.Builder;
import lombok.EqualsAndHashCode;
import lombok.RequiredArgsConstructor;
import lombok.ToString;
import lombok.experimental.Accessors;
import org.apache.commons.beanutils.BeanUtilsBean;
import org.apache.commons.beanutils.ConvertUtilsBean;
import org.junit.Before;
@ -32,11 +30,12 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import lombok.Builder;
import lombok.EqualsAndHashCode;
import lombok.RequiredArgsConstructor;
import lombok.ToString;
import lombok.experimental.Accessors;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.CoreMatchers.nullValue;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.fail;
public class BuilderDynaBeanTest {
@ -109,8 +108,8 @@ public class BuilderDynaBeanTest {
@Test
public void testComplexCreateAllParameters() throws Exception {
TestComplexCreate expected = TestComplexCreate.create("real",
TestSimpleBuilder.builder().stringL1("l1").longVal(10L).build());
TestComplexCreate expected = TestComplexCreate.create(
"real", TestSimpleBuilder.builder().stringL1("l1").longVal(10L).build());
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreate.class, convertUtilsBean);
utilsBean.setProperty(builderDynaBean, "[0]", expected.realName);
@ -136,8 +135,8 @@ public class BuilderDynaBeanTest {
@Test
public void testComplexCreateComplexParameterOnly() throws Exception {
TestComplexCreate expected = TestComplexCreate.create(null,
TestSimpleBuilder.builder().stringL1("l1").longVal(10L).build());
TestComplexCreate expected = TestComplexCreate.create(
null, TestSimpleBuilder.builder().stringL1("l1").longVal(10L).build());
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreate.class, convertUtilsBean);
utilsBean.setProperty(builderDynaBean, "[1].stringL1", expected.test1.stringL1);
@ -161,7 +160,8 @@ public class BuilderDynaBeanTest {
@Test
public void testSimpleBuilderAllParameters() throws Exception {
TestSimpleBuilder expected = TestSimpleBuilder.builder().stringL1("l1").longVal(10L).build();
TestSimpleBuilder expected =
TestSimpleBuilder.builder().stringL1("l1").longVal(10L).build();
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSimpleBuilder.class, convertUtilsBean);
utilsBean.setProperty(builderDynaBean, "stringL1", expected.stringL1);
@ -213,12 +213,14 @@ public class BuilderDynaBeanTest {
@Test
public void testComplexCreateSimpleBuilderVariantAllParameters() throws Exception {
TestSimpleBuilder variant = TestSimpleBuilder.builder().longVal(10L).stringL1("variant").build();
TestSimpleBuilder variant =
TestSimpleBuilder.builder().longVal(10L).stringL1("variant").build();
TestComplexCreateVariance expected = TestComplexCreateVariance.create("simple-builder", variant);
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean);
utilsBean.setProperty(builderDynaBean, "[0]", expected.varianceName);
utilsBean.setProperty(builderDynaBean, "[1].class", expected.variant.getClass().getName());
utilsBean.setProperty(
builderDynaBean, "[1].class", expected.variant.getClass().getName());
utilsBean.setProperty(builderDynaBean, "[1].longVal", variant.longVal);
utilsBean.setProperty(builderDynaBean, "[1].stringL1", variant.stringL1);
@ -229,8 +231,11 @@ public class BuilderDynaBeanTest {
@Test
public void testComplexCreateVariantBuilderAllParameters() throws Exception {
TestVariantBuilder variant = TestVariantBuilder.builder().variantBuilderName("variant-build").intClass(20)
.testEnum(TestEnum.Blue).build();
TestVariantBuilder variant = TestVariantBuilder.builder()
.variantBuilderName("variant-build")
.intClass(20)
.testEnum(TestEnum.Blue)
.build();
TestComplexCreateVariance expected = TestComplexCreateVariance.create("builder-variant", variant);
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean);
@ -264,13 +269,16 @@ public class BuilderDynaBeanTest {
@Test
public void testComplexCreateVariantBuilderAllParametersPrefixWithJoiner() throws Exception {
TestVariantBuilder variant = TestVariantBuilder.builder().variantBuilderName("variant-build").intClass(20)
.testEnum(TestEnum.Blue).build();
TestVariantBuilder variant = TestVariantBuilder.builder()
.variantBuilderName("variant-build")
.intClass(20)
.testEnum(TestEnum.Blue)
.build();
TestComplexCreateVariance expected = TestComplexCreateVariance.create("builder-variant-prefix", variant);
String prefix = variant.getClass().getEnclosingClass().getName() + "$";
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean,
prefix);
BuilderDynaBean builderDynaBean =
new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean, prefix);
utilsBean.setProperty(builderDynaBean, "[0]", expected.varianceName);
utilsBean.setProperty(builderDynaBean, "[1].class", variant.getClass().getSimpleName());
utilsBean.setProperty(builderDynaBean, "[1].variantBuilderName", variant.variantBuilderName);
@ -284,13 +292,16 @@ public class BuilderDynaBeanTest {
@Test
public void testComplexCreateVariantBuilderAllParametersPrefixWithOutJoiner() throws Exception {
TestVariantBuilder variant = TestVariantBuilder.builder().variantBuilderName("variant-build").intClass(20)
.testEnum(TestEnum.Blue).build();
TestVariantBuilder variant = TestVariantBuilder.builder()
.variantBuilderName("variant-build")
.intClass(20)
.testEnum(TestEnum.Blue)
.build();
TestComplexCreateVariance expected = TestComplexCreateVariance.create("builder-variant-prefix", variant);
String prefix = variant.getClass().getEnclosingClass().getName();
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean,
prefix);
BuilderDynaBean builderDynaBean =
new BuilderDynaBean(TestComplexCreateVariance.class, convertUtilsBean, prefix);
utilsBean.setProperty(builderDynaBean, "[0]", expected.varianceName);
utilsBean.setProperty(builderDynaBean, "[1].class", variant.getClass().getSimpleName());
utilsBean.setProperty(builderDynaBean, "[1].variantBuilderName", variant.variantBuilderName);
@ -330,11 +341,21 @@ public class BuilderDynaBeanTest {
@Test
public void testComplexRootAllParameters() throws Exception {
TestSimpleBuilder simpleBuilder = TestSimpleBuilder.builder().stringL1("simple-l1").longVal(20L).build();
TestRootClass expected = TestRootClass.builder().intVal(10).stringVal("root").testEnum(TestEnum.Red)
.testComplexCreate(TestComplexCreate.create("real",
TestSimpleBuilder.builder().stringL1("complex-l1").longVal(10L).build()))
.testSimpleBuilder(simpleBuilder).testSimpleCreate(TestSimpleCreate.create("first", "last")).build();
TestSimpleBuilder simpleBuilder =
TestSimpleBuilder.builder().stringL1("simple-l1").longVal(20L).build();
TestRootClass expected = TestRootClass.builder()
.intVal(10)
.stringVal("root")
.testEnum(TestEnum.Red)
.testComplexCreate(TestComplexCreate.create(
"real",
TestSimpleBuilder.builder()
.stringL1("complex-l1")
.longVal(10L)
.build()))
.testSimpleBuilder(simpleBuilder)
.testSimpleCreate(TestSimpleCreate.create("first", "last"))
.build();
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestRootClass.class, convertUtilsBean);
@ -342,10 +363,10 @@ public class BuilderDynaBeanTest {
utilsBean.setProperty(builderDynaBean, "stringVal", expected.stringVal);
utilsBean.setProperty(builderDynaBean, "testEnum", expected.testEnum);
utilsBean.setProperty(builderDynaBean, "testComplexCreate.[0]", expected.testComplexCreate.realName);
utilsBean.setProperty(builderDynaBean, "testComplexCreate.[1].stringL1",
expected.testComplexCreate.test1.stringL1);
utilsBean.setProperty(builderDynaBean, "testComplexCreate.[1].longVal",
expected.testComplexCreate.test1.longVal);
utilsBean.setProperty(
builderDynaBean, "testComplexCreate.[1].stringL1", expected.testComplexCreate.test1.stringL1);
utilsBean.setProperty(
builderDynaBean, "testComplexCreate.[1].longVal", expected.testComplexCreate.test1.longVal);
utilsBean.setProperty(builderDynaBean, "testSimpleBuilder.class", TestSimpleBuilder.class.getName());
utilsBean.setProperty(builderDynaBean, "testSimpleBuilder.stringL1", simpleBuilder.stringL1);
utilsBean.setProperty(builderDynaBean, "testSimpleBuilder.longVal", simpleBuilder.longVal);
@ -370,7 +391,11 @@ public class BuilderDynaBeanTest {
@Test
public void testComplexRootTopLevelOnly() throws Exception {
TestRootClass expected = TestRootClass.builder().intVal(10).stringVal("root").testEnum(TestEnum.Red).build();
TestRootClass expected = TestRootClass.builder()
.intVal(10)
.stringVal("root")
.testEnum(TestEnum.Red)
.build();
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestRootClass.class, convertUtilsBean);
@ -385,12 +410,17 @@ public class BuilderDynaBeanTest {
@Test
public void testSupplierNotUsed() throws Exception {
TestVariantBuilder variant = TestVariantBuilder.builder().testEnum(TestEnum.Green).intClass(10)
.variantBuilderName("variant-supplier").build();
TestSupplierClass expected = TestSupplierClass.builder().variantClass(variant).build();
TestVariantBuilder variant = TestVariantBuilder.builder()
.testEnum(TestEnum.Green)
.intClass(10)
.variantBuilderName("variant-supplier")
.build();
TestSupplierClass expected =
TestSupplierClass.builder().variantClass(variant).build();
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSupplierClass.class, convertUtilsBean);
utilsBean.setProperty(builderDynaBean, "variantClass.class", variant.getClass().getName());
utilsBean.setProperty(
builderDynaBean, "variantClass.class", variant.getClass().getName());
utilsBean.setProperty(builderDynaBean, "variantClass.testEnum", variant.testEnum);
utilsBean.setProperty(builderDynaBean, "variantClass.intClass", variant.intClass);
utilsBean.setProperty(builderDynaBean, "variantClass.variantBuilderName", variant.variantBuilderName);
@ -422,8 +452,11 @@ public class BuilderDynaBeanTest {
@Test
public void testVariantBuildsToSuperType() throws Exception {
TestVariantBuilder expected = TestVariantBuilder.builder().intClass(10).testEnum(TestEnum.Green)
.variantBuilderName("variant-super").build();
TestVariantBuilder expected = TestVariantBuilder.builder()
.intClass(10)
.testEnum(TestEnum.Green)
.variantBuilderName("variant-super")
.build();
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestInterface.class, convertUtilsBean);
utilsBean.setProperty(builderDynaBean, "class", expected.getClass().getName());
@ -439,9 +472,11 @@ public class BuilderDynaBeanTest {
@Test
public void testEmptyPropertyHandler() throws Exception {
String emptyPropertyValue = "test-property";
TestVariantCreate expected = TestVariantCreate.create(emptyPropertyValue, (long) emptyPropertyValue.length(),
emptyPropertyValue + "-vary");
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestInterface.class, convertUtilsBean,
TestVariantCreate expected = TestVariantCreate.create(
emptyPropertyValue, (long) emptyPropertyValue.length(), emptyPropertyValue + "-vary");
BuilderDynaBean builderDynaBean = new BuilderDynaBean(
TestInterface.class,
convertUtilsBean,
s -> TestVariantCreate.create(s, (long) s.length(), s + "-vary"));
utilsBean.setProperty(builderDynaBean, "", emptyPropertyValue);
@ -455,8 +490,8 @@ public class BuilderDynaBeanTest {
thrown.expect(IllegalStateException.class);
thrown.expectMessage(containsString("When a property handler is resolved further properties may not be set."));
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestInterface.class, convertUtilsBean,
s -> TestVariantCreate.create("test", 10, "test"));
BuilderDynaBean builderDynaBean = new BuilderDynaBean(
TestInterface.class, convertUtilsBean, s -> TestVariantCreate.create("test", 10, "test"));
utilsBean.setProperty(builderDynaBean, "", "test");
utilsBean.setProperty(builderDynaBean, "[0]", "test");
}
@ -468,8 +503,8 @@ public class BuilderDynaBeanTest {
thrown.expectMessage(containsString(TestInterface.class.getName()));
thrown.expectMessage(containsString("cannot be assigned to"));
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestInterface.class, convertUtilsBean,
s -> TestEnum.Green);
BuilderDynaBean builderDynaBean =
new BuilderDynaBean(TestInterface.class, convertUtilsBean, s -> TestEnum.Green);
utilsBean.setProperty(builderDynaBean, "", "test");
@ -478,8 +513,11 @@ public class BuilderDynaBeanTest {
@Test
public void testSimpleArrayValues() throws Exception {
SimpleArrayClassVariant expected = SimpleArrayClassVariant.builder().ints(new Integer[] { 1, 2, 3 })
.variantName("simple-array").longs(new Long[] { 1L, 2L, 3L }).strings(new String[] { "a", "b", "c" })
SimpleArrayClassVariant expected = SimpleArrayClassVariant.builder()
.ints(new Integer[] {1, 2, 3})
.variantName("simple-array")
.longs(new Long[] {1L, 2L, 3L})
.strings(new String[] {"a", "b", "c"})
.build();
BuilderDynaBean builderDynaBean = new BuilderDynaBean(SimpleArrayClassVariant.class, convertUtilsBean);
@ -503,12 +541,20 @@ public class BuilderDynaBeanTest {
@Test
public void testComplexArrayValuesBuilder() throws Exception {
TestVariantBuilder variant1 = TestVariantBuilder.builder().variantBuilderName("variant-1")
.testEnum(TestEnum.Green).intClass(10).build();
TestVariantBuilder variant2 = TestVariantBuilder.builder().variantBuilderName("variant-2")
.testEnum(TestEnum.Blue).intClass(20).build();
ComplexArrayClassVariant expected = ComplexArrayClassVariant.builder().variantName("complex-test")
.tests(new TestInterface[] { variant1, variant2 }).build();
TestVariantBuilder variant1 = TestVariantBuilder.builder()
.variantBuilderName("variant-1")
.testEnum(TestEnum.Green)
.intClass(10)
.build();
TestVariantBuilder variant2 = TestVariantBuilder.builder()
.variantBuilderName("variant-2")
.testEnum(TestEnum.Blue)
.intClass(20)
.build();
ComplexArrayClassVariant expected = ComplexArrayClassVariant.builder()
.variantName("complex-test")
.tests(new TestInterface[] {variant1, variant2})
.build();
BuilderDynaBean builderDynaBean = new BuilderDynaBean(ComplexArrayClassVariant.class, convertUtilsBean);
@ -533,18 +579,22 @@ public class BuilderDynaBeanTest {
TestVariantCreate variant1 = TestVariantCreate.create("variant-1", 10L, "vary-1");
TestVariantCreate variant2 = TestVariantCreate.create("variant-2", 20L, "vary-2");
ComplexArrayClassVariant expected = ComplexArrayClassVariant.builder().variantName("create-test")
.tests(new TestInterface[] { variant1, variant2 }).build();
ComplexArrayClassVariant expected = ComplexArrayClassVariant.builder()
.variantName("create-test")
.tests(new TestInterface[] {variant1, variant2})
.build();
BuilderDynaBean builderDynaBean = new BuilderDynaBean(ComplexArrayClassVariant.class, convertUtilsBean);
utilsBean.setProperty(builderDynaBean, "variantName", expected.variantName);
utilsBean.setProperty(builderDynaBean, "tests[0].class", variant1.getClass().getName());
utilsBean.setProperty(
builderDynaBean, "tests[0].class", variant1.getClass().getName());
utilsBean.setProperty(builderDynaBean, "tests[0].[0]", variant1.variantCreateName);
utilsBean.setProperty(builderDynaBean, "tests[0].[1]", variant1.longClass);
utilsBean.setProperty(builderDynaBean, "tests[0].[2]", variant1.varyString);
utilsBean.setProperty(builderDynaBean, "tests[1].class", variant2.getClass().getName());
utilsBean.setProperty(
builderDynaBean, "tests[1].class", variant2.getClass().getName());
utilsBean.setProperty(builderDynaBean, "tests[1].[0]", variant2.variantCreateName);
utilsBean.setProperty(builderDynaBean, "tests[1].[1]", variant2.longClass);
utilsBean.setProperty(builderDynaBean, "tests[1].[2]", variant2.varyString);
@ -552,7 +602,6 @@ public class BuilderDynaBeanTest {
ComplexArrayClassVariant actual = builderDynaBean.build(ComplexArrayClassVariant.class);
assertThat(actual, equalTo(expected));
}
@Test
@ -562,13 +611,18 @@ public class BuilderDynaBeanTest {
if (i % 2 == 0) {
variants[i] = TestVariantCreate.create("create-variant-" + i, i + 5, "vary-" + i);
} else {
variants[i] = TestVariantBuilder.builder().testEnum(TestEnum.values()[i % TestEnum.values().length])
.intClass(i).variantBuilderName("builder-variant-" + i).build();
variants[i] = TestVariantBuilder.builder()
.testEnum(TestEnum.values()[i % TestEnum.values().length])
.intClass(i)
.variantBuilderName("builder-variant-" + i)
.build();
}
}
ComplexArrayClassVariant expected = ComplexArrayClassVariant.builder().variantName("large-complex")
.tests(variants).build();
ComplexArrayClassVariant expected = ComplexArrayClassVariant.builder()
.variantName("large-complex")
.tests(variants)
.build();
BuilderDynaBean builderDynaBean = new BuilderDynaBean(ComplexArrayClassVariant.class, convertUtilsBean);
@ -578,13 +632,15 @@ public class BuilderDynaBeanTest {
TestInterface variant = variants[i];
if (variant instanceof TestVariantCreate) {
TestVariantCreate create = (TestVariantCreate) variant;
utilsBean.setProperty(builderDynaBean, prefix + "class", create.getClass().getName());
utilsBean.setProperty(
builderDynaBean, prefix + "class", create.getClass().getName());
utilsBean.setProperty(builderDynaBean, prefix + "[0]", create.variantCreateName);
utilsBean.setProperty(builderDynaBean, prefix + "[1]", create.longClass);
utilsBean.setProperty(builderDynaBean, prefix + "[2]", create.varyString);
} else if (variant instanceof TestVariantBuilder) {
TestVariantBuilder builder = (TestVariantBuilder) variant;
utilsBean.setProperty(builderDynaBean, prefix + "class", builder.getClass().getName());
utilsBean.setProperty(
builderDynaBean, prefix + "class", builder.getClass().getName());
utilsBean.setProperty(builderDynaBean, prefix + "variantBuilderName", builder.variantBuilderName);
utilsBean.setProperty(builderDynaBean, prefix + "intClass", builder.intClass);
utilsBean.setProperty(builderDynaBean, prefix + "testEnum", builder.testEnum);
@ -667,25 +723,27 @@ public class BuilderDynaBeanTest {
@Test
public void testAdditionalMutators() throws Exception {
TestSimpleBuilder expected = TestSimpleBuilder.builder().stringL1("test").longVal(10L).build();
TestSimpleBuilder expected =
TestSimpleBuilder.builder().stringL1("test").longVal(10L).build();
BuilderDynaBean builderDynaBean = new BuilderDynaBean(TestSimpleBuilder.class, convertUtilsBean);
utilsBean.setProperty(builderDynaBean, "stringL1", expected.stringL1);
TestSimpleBuilder actual = builderDynaBean.build(TestSimpleBuilder.class,
b -> ((TestSimpleBuilder.TestSimpleBuilderBuilder) b).longVal(expected.longVal));
TestSimpleBuilder actual =
builderDynaBean.build(TestSimpleBuilder.class, b -> ((TestSimpleBuilder.TestSimpleBuilderBuilder) b)
.longVal(expected.longVal));
assertThat(actual, equalTo(expected));
}
public enum TestEnum {
Red, Green, Blue
Red,
Green,
Blue
}
public interface TestInterface {
}
public interface TestInterface {}
@Accessors(fluent = true)
@ToString
@ -838,7 +896,5 @@ public class BuilderDynaBeanTest {
}
public String name = "default";
}
}
}

102
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/ConfigurationSettableUtilsTest.java
View file

@ -15,18 +15,17 @@
package software.amazon.kinesis.multilang.config;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.junit.Assert.assertThat;
import java.util.Optional;
import org.junit.Test;
import lombok.Builder;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.Setter;
import lombok.experimental.Accessors;
import org.junit.Test;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.junit.Assert.assertThat;
public class ConfigurationSettableUtilsTest {
@ -44,7 +43,10 @@ public class ConfigurationSettableUtilsTest {
public void testPrimitivesSet() {
ConfigResult expected = ConfigResult.builder().rawInt(10).rawLong(15L).build();
ConfigObject configObject = ConfigObject.builder().rawInt(expected.rawInt).rawLong(expected.rawLong).build();
ConfigObject configObject = ConfigObject.builder()
.rawInt(expected.rawInt)
.rawLong(expected.rawLong)
.build();
ConfigResult actual = resolve(configObject);
assertThat(actual, equalTo(expected));
@ -52,10 +54,14 @@ public class ConfigurationSettableUtilsTest {
@Test
public void testHeapValuesSet() {
ConfigResult expected = ConfigResult.builder().name("test").boxedInt(10).boxedLong(15L).build();
ConfigResult expected =
ConfigResult.builder().name("test").boxedInt(10).boxedLong(15L).build();
ConfigObject configObject = ConfigObject.builder().name(expected.name).boxedInt(expected.boxedInt.intValue())
.boxedLong(expected.boxedLong.longValue()).build();
ConfigObject configObject = ConfigObject.builder()
.name(expected.name)
.boxedInt(expected.boxedInt.intValue())
.boxedLong(expected.boxedLong.longValue())
.build();
ConfigResult actual = resolve(configObject);
assertThat(actual, equalTo(expected));
@ -63,27 +69,39 @@ public class ConfigurationSettableUtilsTest {
@Test
public void testComplexValuesSet() {
ComplexValue complexValue = ComplexValue.builder().name("complex").value(10).build();
ConfigResult expected = ConfigResult.builder().complexValue(complexValue).build();
ComplexValue complexValue =
ComplexValue.builder().name("complex").value(10).build();
ConfigResult expected =
ConfigResult.builder().complexValue(complexValue).build();
ConfigObject configObject = ConfigObject.builder()
.complexValue(ComplexValue.builder().name(complexValue.name).value(complexValue.value).build()).build();
.complexValue(ComplexValue.builder()
.name(complexValue.name)
.value(complexValue.value)
.build())
.build();
ConfigResult actual = resolve(configObject);
assertThat(actual, equalTo(expected));
}
@Test
public void testOptionalValuesSet() {
ComplexValue complexValue = ComplexValue.builder().name("optional-complex").value(20).build();
ConfigResult expected = ConfigResult.builder().optionalString(Optional.of("test"))
.optionalInteger(Optional.of(10)).optionalLong(Optional.of(15L))
.optionalComplexValue(Optional.of(complexValue)).build();
ComplexValue complexValue =
ComplexValue.builder().name("optional-complex").value(20).build();
ConfigResult expected = ConfigResult.builder()
.optionalString(Optional.of("test"))
.optionalInteger(Optional.of(10))
.optionalLong(Optional.of(15L))
.optionalComplexValue(Optional.of(complexValue))
.build();
ConfigObject configObject = ConfigObject.builder().optionalString(expected.optionalString.get())
.optionalInteger(expected.optionalInteger.get()).optionalLong(expected.optionalLong.get())
.optionalComplexValue(expected.optionalComplexValue.get()).build();
ConfigObject configObject = ConfigObject.builder()
.optionalString(expected.optionalString.get())
.optionalInteger(expected.optionalInteger.get())
.optionalLong(expected.optionalLong.get())
.optionalComplexValue(expected.optionalComplexValue.get())
.build();
ConfigResult actual = resolve(configObject);
assertThat(actual, equalTo(expected));
@ -91,20 +109,29 @@ public class ConfigurationSettableUtilsTest {
@Test
public void testRenamedRawValues() {
ComplexValue complexValue = ComplexValue.builder().name("renamed-complex").value(20).build();
ConfigResult expected = ConfigResult.builder().renamedString("renamed").renamedInt(10)
.renamedOptionalString(Optional.of("renamed-optional")).renamedComplexValue(complexValue).build();
ComplexValue complexValue =
ComplexValue.builder().name("renamed-complex").value(20).build();
ConfigResult expected = ConfigResult.builder()
.renamedString("renamed")
.renamedInt(10)
.renamedOptionalString(Optional.of("renamed-optional"))
.renamedComplexValue(complexValue)
.build();
ConfigObject configObject = ConfigObject.builder().toRenameString(expected.renamedString)
.toRenameInt(expected.renamedInt).toRenameComplexValue(complexValue)
.optionalToRename(expected.renamedOptionalString.get()).build();
ConfigObject configObject = ConfigObject.builder()
.toRenameString(expected.renamedString)
.toRenameInt(expected.renamedInt)
.toRenameComplexValue(complexValue)
.optionalToRename(expected.renamedOptionalString.get())
.build();
ConfigResult actual = resolve(configObject);
assertThat(actual, equalTo(expected));
}
private ConfigResult resolve(ConfigObject configObject) {
return ConfigurationSettableUtils.resolveFields(configObject, ConfigResult.builder().build());
return ConfigurationSettableUtils.resolveFields(
configObject, ConfigResult.builder().build());
}
@Accessors(fluent = true)
@ -129,7 +156,6 @@ public class ConfigurationSettableUtilsTest {
private int renamedInt;
private Optional<String> renamedOptionalString;
private ComplexValue renamedComplexValue;
}
@Accessors(fluent = true)
@ -145,35 +171,47 @@ public class ConfigurationSettableUtilsTest {
@ConfigurationSettable(configurationClass = ConfigResult.class)
private String name;
@ConfigurationSettable(configurationClass = ConfigResult.class)
private int rawInt;
@ConfigurationSettable(configurationClass = ConfigResult.class)
private Integer boxedInt;
@ConfigurationSettable(configurationClass = ConfigResult.class)
private long rawLong;
@ConfigurationSettable(configurationClass = ConfigResult.class)
private Long boxedLong;
@ConfigurationSettable(configurationClass = ConfigResult.class)
private ComplexValue complexValue;
@ConfigurationSettable(configurationClass = ConfigResult.class, convertToOptional = true)
private String optionalString;
@ConfigurationSettable(configurationClass = ConfigResult.class, convertToOptional = true)
private Integer optionalInteger;
@ConfigurationSettable(configurationClass = ConfigResult.class, convertToOptional = true)
private Long optionalLong;
@ConfigurationSettable(configurationClass = ConfigResult.class, convertToOptional = true)
private ComplexValue optionalComplexValue;
@ConfigurationSettable(configurationClass = ConfigResult.class, methodName = "renamedString")
private String toRenameString;
@ConfigurationSettable(configurationClass = ConfigResult.class, methodName = "renamedInt")
private int toRenameInt;
@ConfigurationSettable(configurationClass = ConfigResult.class, methodName = "renamedOptionalString", convertToOptional = true)
@ConfigurationSettable(
configurationClass = ConfigResult.class,
methodName = "renamedOptionalString",
convertToOptional = true)
private String optionalToRename;
@ConfigurationSettable(configurationClass = ConfigResult.class, methodName = "renamedComplexValue")
private ComplexValue toRenameComplexValue;
}
}
}

4
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/DatePropertyValueDecoderTest.java
View file

@ -14,12 +14,12 @@
*/
package software.amazon.kinesis.multilang.config;
import static org.junit.Assert.assertEquals;
import java.util.Date;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
public class DatePropertyValueDecoderTest {
private DatePropertyValueDecoder decoder = new DatePropertyValueDecoder();

19
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/FanoutConfigBeanTest.java
View file

@ -21,7 +21,6 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
import software.amazon.awssdk.services.kinesis.KinesisAsyncClient;
import software.amazon.kinesis.retrieval.fanout.FanOutConfig;
@ -50,18 +49,22 @@ public class FanoutConfigBeanTest {
MultiLangDaemonConfiguration configuration = new MultiLangDaemonConfiguration(utilsBean, convertUtilsBean);
configuration.setStreamName("test-stream");
configuration.setApplicationName("test-application");
FanOutConfig fanOutConfig =fanoutConfigBean.build(kinesisAsyncClient, configuration);
FanOutConfig fanOutConfig = fanoutConfigBean.build(kinesisAsyncClient, configuration);
assertThat(fanOutConfig.kinesisClient(), equalTo(kinesisAsyncClient));
assertThat(fanOutConfig.streamName(), equalTo(configuration.getStreamName()));
assertThat(fanOutConfig.applicationName(), equalTo(configuration.getApplicationName()));
assertThat(fanOutConfig.consumerArn(), equalTo(fanoutConfigBean.getConsumerArn()));
assertThat(fanOutConfig.consumerName(), equalTo(fanoutConfigBean.getConsumerName()));
assertThat(fanOutConfig.maxDescribeStreamConsumerRetries(), equalTo(fanoutConfigBean.getMaxDescribeStreamConsumerRetries()));
assertThat(fanOutConfig.maxDescribeStreamSummaryRetries(), equalTo(fanoutConfigBean.getMaxDescribeStreamSummaryRetries()));
assertThat(fanOutConfig.registerStreamConsumerRetries(), equalTo(fanoutConfigBean.getRegisterStreamConsumerRetries()));
assertThat(
fanOutConfig.maxDescribeStreamConsumerRetries(),
equalTo(fanoutConfigBean.getMaxDescribeStreamConsumerRetries()));
assertThat(
fanOutConfig.maxDescribeStreamSummaryRetries(),
equalTo(fanoutConfigBean.getMaxDescribeStreamSummaryRetries()));
assertThat(
fanOutConfig.registerStreamConsumerRetries(),
equalTo(fanoutConfigBean.getRegisterStreamConsumerRetries()));
assertThat(fanOutConfig.retryBackoffMillis(), equalTo(fanoutConfigBean.getRetryBackoffMillis()));
}
}
}

486
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/KinesisClientLibConfiguratorTest.java
View file

@ -14,15 +14,6 @@
*/
package software.amazon.kinesis.multilang.config;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.nullValue;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.net.URI;
@ -34,42 +25,45 @@ import java.util.Set;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.google.common.collect.ImmutableSet;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.junit.Rule;
import org.junit.Test;
import com.google.common.collect.ImmutableSet;
import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.kinesis.common.InitialPositionInStream;
import software.amazon.kinesis.metrics.MetricsLevel;
import software.amazon.kinesis.processor.ShardRecordProcessorFactory;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.nullValue;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@RunWith(MockitoJUnitRunner.class)
public class KinesisClientLibConfiguratorTest {
private String credentialName1 = "software.amazon.kinesis.multilang.config.KinesisClientLibConfiguratorTest$AlwaysSucceedCredentialsProvider";
private String credentialName2 = "software.amazon.kinesis.multilang.config.KinesisClientLibConfiguratorTest$AlwaysFailCredentialsProvider";
private String credentialNameKinesis = "software.amazon.kinesis.multilang.config.KinesisClientLibConfiguratorTest$AlwaysSucceedCredentialsProviderKinesis";
private String credentialNameDynamoDB = "software.amazon.kinesis.multilang.config.KinesisClientLibConfiguratorTest$AlwaysSucceedCredentialsProviderDynamoDB";
private String credentialNameCloudWatch = "software.amazon.kinesis.multilang.config.KinesisClientLibConfiguratorTest$AlwaysSucceedCredentialsProviderCloudWatch";
private KinesisClientLibConfigurator configurator = new KinesisClientLibConfigurator();
@Rule
public final ExpectedException thrown = ExpectedException.none();
@Mock
private ShardRecordProcessorFactory shardRecordProcessorFactory;
private final String credentialName1 = AlwaysSucceedCredentialsProvider.class.getName();
private final String credentialName2 = AlwaysFailCredentialsProvider.class.getName();
private final String credentialNameKinesis = AlwaysSucceedCredentialsProviderKinesis.class.getName();
private final String credentialNameDynamoDB = AlwaysSucceedCredentialsProviderDynamoDB.class.getName();
private final String credentialNameCloudWatch = AlwaysSucceedCredentialsProviderCloudWatch.class.getName();
private final KinesisClientLibConfigurator configurator = new KinesisClientLibConfigurator();
@Test
public void testWithBasicSetup() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a",
"applicationName = b", "AWSCredentialsProvider = " + credentialName1, "workerId = 123" }, '\n'));
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = " + credentialName1,
"workerId = 123"
},
'\n'));
assertEquals(config.getApplicationName(), "b");
assertEquals(config.getStreamName(), "a");
assertEquals(config.getWorkerIdentifier(), "123");
@ -79,9 +73,16 @@ public class KinesisClientLibConfiguratorTest {
@Test
public void testWithLongVariables() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "applicationName = app",
"streamName = 123", "AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
"workerId = 123", "failoverTimeMillis = 100", "shardSyncIntervalMillis = 500" }, '\n'));
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
new String[] {
"applicationName = app",
"streamName = 123",
"AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
"workerId = 123",
"failoverTimeMillis = 100",
"shardSyncIntervalMillis = 500"
},
'\n'));
assertEquals(config.getApplicationName(), "app");
assertEquals(config.getStreamName(), "123");
@ -93,9 +94,14 @@ public class KinesisClientLibConfiguratorTest {
@Test
public void testWithInitialPositionInStreamExtended() {
long epochTimeInSeconds = 1617406032;
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "applicationName = app",
"streamName = 123", "AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
"initialPositionInStreamExtended = " + epochTimeInSeconds}, '\n'));
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
new String[] {
"applicationName = app",
"streamName = 123",
"AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
"initialPositionInStreamExtended = " + epochTimeInSeconds
},
'\n'));
assertEquals(config.getInitialPositionInStreamExtended().getTimestamp(), new Date(epochTimeInSeconds * 1000L));
assertEquals(config.getInitialPositionInStream(), InitialPositionInStream.AT_TIMESTAMP);
@ -106,9 +112,14 @@ public class KinesisClientLibConfiguratorTest {
// AT_TIMESTAMP cannot be used as initialPositionInStream. If a user wants to specify AT_TIMESTAMP,
// they must specify the time with initialPositionInStreamExtended.
try {
getConfiguration(StringUtils.join(new String[] { "applicationName = app",
"streamName = 123", "AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
"initialPositionInStream = AT_TIMESTAMP"}, '\n'));
getConfiguration(StringUtils.join(
new String[] {
"applicationName = app",
"streamName = 123",
"AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
"initialPositionInStream = AT_TIMESTAMP"
},
'\n'));
fail("Should have thrown when initialPositionInStream is set to AT_TIMESTAMP");
} catch (Exception e) {
Throwable rootCause = ExceptionUtils.getRootCause(e);
@ -121,9 +132,14 @@ public class KinesisClientLibConfiguratorTest {
// initialPositionInStreamExtended takes a long value indicating seconds since epoch. If a non-long
// value is provided, the constructor should throw an IllegalArgumentException exception.
try {
getConfiguration(StringUtils.join(new String[] { "applicationName = app",
"streamName = 123", "AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
"initialPositionInStreamExtended = null"}, '\n'));
getConfiguration(StringUtils.join(
new String[] {
"applicationName = app",
"streamName = 123",
"AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
"initialPositionInStreamExtended = null"
},
'\n'));
fail("Should have thrown when initialPositionInStreamExtended is set to null");
} catch (Exception e) {
Throwable rootCause = ExceptionUtils.getRootCause(e);
@ -134,8 +150,13 @@ public class KinesisClientLibConfiguratorTest {
@Test
public void testWithUnsupportedClientConfigurationVariables() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
new String[] { "AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2, "workerId = id",
"kinesisClientConfig = {}", "streamName = stream", "applicationName = b" },
new String[] {
"AWSCredentialsProvider = " + credentialName1 + ", " + credentialName2,
"workerId = id",
"kinesisClientConfig = {}",
"streamName = stream",
"applicationName = b"
},
'\n'));
assertEquals(config.getApplicationName(), "b");
@ -146,10 +167,18 @@ public class KinesisClientLibConfiguratorTest {
@Test
public void testWithIntVariables() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = kinesis",
"AWSCredentialsProvider = " + credentialName2 + ", " + credentialName1, "workerId = w123",
"maxRecords = 10", "metricsMaxQueueSize = 20", "applicationName = kinesis",
"retryGetRecordsInSeconds = 2", "maxGetRecordsThreadPool = 1" }, '\n'));
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
new String[] {
"streamName = kinesis",
"AWSCredentialsProvider = " + credentialName2 + ", " + credentialName1,
"workerId = w123",
"maxRecords = 10",
"metricsMaxQueueSize = 20",
"applicationName = kinesis",
"retryGetRecordsInSeconds = 2",
"maxGetRecordsThreadPool = 1"
},
'\n'));
assertEquals(config.getApplicationName(), "kinesis");
assertEquals(config.getStreamName(), "kinesis");
@ -162,9 +191,15 @@ public class KinesisClientLibConfiguratorTest {
@Test
public void testWithBooleanVariables() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a",
"applicationName = b", "AWSCredentialsProvider = ABCD, " + credentialName1, "workerId = 0",
"cleanupLeasesUponShardCompletion = false", "validateSequenceNumberBeforeCheckpointing = true" },
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = ABCD, " + credentialName1,
"workerId = 0",
"cleanupLeasesUponShardCompletion = false",
"validateSequenceNumberBeforeCheckpointing = true"
},
'\n'));
assertEquals(config.getApplicationName(), "b");
@ -176,9 +211,16 @@ public class KinesisClientLibConfiguratorTest {
@Test
public void testWithStringVariables() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a",
"applicationName = b", "AWSCredentialsProvider = ABCD," + credentialName1, "workerId = 1",
"kinesisEndpoint = https://kinesis", "metricsLevel = SUMMARY" }, '\n'));
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = ABCD," + credentialName1,
"workerId = 1",
"kinesisEndpoint = https://kinesis",
"metricsLevel = SUMMARY"
},
'\n'));
assertEquals(config.getWorkerIdentifier(), "1");
assertEquals(config.getKinesisClient().get("endpointOverride"), URI.create("https://kinesis"));
@ -187,38 +229,66 @@ public class KinesisClientLibConfiguratorTest {
@Test
public void testWithSetVariables() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a",
"applicationName = b", "AWSCredentialsProvider = ABCD," + credentialName1, "workerId = 1",
"metricsEnabledDimensions = ShardId, WorkerIdentifier" }, '\n'));
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = ABCD," + credentialName1,
"workerId = 1",
"metricsEnabledDimensions = ShardId, WorkerIdentifier"
},
'\n'));
Set<String> expectedMetricsEnabledDimensions = ImmutableSet.<String> builder()
.add("ShardId", "WorkerIdentifier").build();
assertThat(new HashSet<>(Arrays.asList(config.getMetricsEnabledDimensions())), equalTo(expectedMetricsEnabledDimensions));
Set<String> expectedMetricsEnabledDimensions = ImmutableSet.<String>builder()
.add("ShardId", "WorkerIdentifier")
.build();
assertThat(
new HashSet<>(Arrays.asList(config.getMetricsEnabledDimensions())),
equalTo(expectedMetricsEnabledDimensions));
}
@Test
public void testWithInitialPositionInStreamTrimHorizon() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a",
"applicationName = b", "AWSCredentialsProvider = ABCD," + credentialName1, "workerId = 123",
"initialPositionInStream = TriM_Horizon" }, '\n'));
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = ABCD," + credentialName1,
"workerId = 123",
"initialPositionInStream = TriM_Horizon"
},
'\n'));
assertEquals(config.getInitialPositionInStream(), InitialPositionInStream.TRIM_HORIZON);
}
@Test
public void testWithInitialPositionInStreamLatest() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a",
"applicationName = b", "AWSCredentialsProvider = ABCD," + credentialName1, "workerId = 123",
"initialPositionInStream = LateSt" }, '\n'));
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = ABCD," + credentialName1,
"workerId = 123",
"initialPositionInStream = LateSt"
},
'\n'));
assertEquals(config.getInitialPositionInStream(), InitialPositionInStream.LATEST);
}
@Test
public void testSkippingNonKCLVariables() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a",
"applicationName = b", "AWSCredentialsProvider = ABCD," + credentialName1, "workerId = 123",
"initialPositionInStream = TriM_Horizon", "abc = 1" }, '\n'));
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = ABCD," + credentialName1,
"workerId = 123",
"initialPositionInStream = TriM_Horizon",
"abc = 1"
},
'\n'));
assertEquals(config.getApplicationName(), "b");
assertEquals(config.getStreamName(), "a");
@ -228,118 +298,159 @@ public class KinesisClientLibConfiguratorTest {
@Test
public void testEmptyOptionalVariables() {
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(new String[] { "streamName = a",
"applicationName = b", "AWSCredentialsProvider = ABCD," + credentialName1, "workerId = 123",
"initialPositionInStream = TriM_Horizon", "maxGetRecordsThreadPool = 1" }, '\n'));
MultiLangDaemonConfiguration config = getConfiguration(StringUtils.join(
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = ABCD," + credentialName1,
"workerId = 123",
"initialPositionInStream = TriM_Horizon",
"maxGetRecordsThreadPool = 1"
},
'\n'));
assertThat(config.getMaxGetRecordsThreadPool(), equalTo(1));
assertThat(config.getRetryGetRecordsInSeconds(), nullValue());
}
@Test
public void testWithZeroValue() {
String test = StringUtils.join(new String[] { "streamName = a", "applicationName = b",
"AWSCredentialsProvider = ABCD," + credentialName1, "workerId = 123",
"initialPositionInStream = TriM_Horizon", "maxGetRecordsThreadPool = 0",
"retryGetRecordsInSeconds = 0" }, '\n');
InputStream input = new ByteArrayInputStream(test.getBytes());
try {
configurator.getConfiguration(input);
} catch (Exception e) {
fail("Don't expect to fail on invalid variable value");
}
String test = StringUtils.join(
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = ABCD," + credentialName1,
"workerId = 123",
"initialPositionInStream = TriM_Horizon",
"maxGetRecordsThreadPool = 0",
"retryGetRecordsInSeconds = 0"
},
'\n');
getConfiguration(test);
}
@Test
public void testWithInvalidIntValue() {
String test = StringUtils.join(new String[] { "streamName = a", "applicationName = b",
"AWSCredentialsProvider = " + credentialName1, "workerId = 123", "failoverTimeMillis = 100nf" }, '\n');
InputStream input = new ByteArrayInputStream(test.getBytes());
try {
configurator.getConfiguration(input);
} catch (Exception e) {
fail("Don't expect to fail on invalid variable value");
}
String test = StringUtils.join(
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = " + credentialName1,
"workerId = 123",
"failoverTimeMillis = 100nf"
},
'\n');
getConfiguration(test);
}
@Test
public void testWithNegativeIntValue() {
String test = StringUtils.join(new String[] { "streamName = a", "applicationName = b",
"AWSCredentialsProvider = " + credentialName1, "workerId = 123", "failoverTimeMillis = -12" }, '\n');
InputStream input = new ByteArrayInputStream(test.getBytes());
String test = StringUtils.join(
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = " + credentialName1,
"workerId = 123",
"failoverTimeMillis = -12"
},
'\n');
// separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement
try {
configurator.getConfiguration(input);
} catch (Exception e) {
fail("Don't expect to fail on invalid variable value");
}
getConfiguration(test);
}
@Test
@Test(expected = IllegalArgumentException.class)
public void testWithMissingCredentialsProvider() {
thrown.expect(IllegalArgumentException.class);
thrown.expectMessage("A basic set of AWS credentials must be provided");
String test = StringUtils.join(new String[] { "streamName = a", "applicationName = b", "workerId = 123",
"failoverTimeMillis = 100", "shardSyncIntervalMillis = 500" }, '\n');
InputStream input = new ByteArrayInputStream(test.getBytes());
String test = StringUtils.join(
new String[] {
"streamName = a",
"applicationName = b",
"workerId = 123",
"failoverTimeMillis = 100",
"shardSyncIntervalMillis = 500"
},
'\n');
// separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement
configurator.getConfiguration(input);
getConfiguration(test);
}
@Test
public void testWithMissingWorkerId() {
String test = StringUtils.join(
new String[] { "streamName = a", "applicationName = b", "AWSCredentialsProvider = " + credentialName1,
"failoverTimeMillis = 100", "shardSyncIntervalMillis = 500" },
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = " + credentialName1,
"failoverTimeMillis = 100",
"shardSyncIntervalMillis = 500"
},
'\n');
InputStream input = new ByteArrayInputStream(test.getBytes());
MultiLangDaemonConfiguration config = configurator.getConfiguration(input);
MultiLangDaemonConfiguration config = getConfiguration(test);
// if workerId is not provided, configurator should assign one for it automatically
assertNotNull(config.getWorkerIdentifier());
assertFalse(config.getWorkerIdentifier().isEmpty());
}
@Test
public void testWithMissingStreamName() {
thrown.expect(NullPointerException.class);
thrown.expectMessage("Stream name is required");
String test = StringUtils.join(new String[] { "applicationName = b",
"AWSCredentialsProvider = " + credentialName1, "workerId = 123", "failoverTimeMillis = 100" }, '\n');
InputStream input = new ByteArrayInputStream(test.getBytes());
configurator.getConfiguration(input);
@Test(expected = NullPointerException.class)
public void testWithMissingStreamNameAndMissingStreamArn() {
String test = StringUtils.join(
new String[] {
"applicationName = b",
"AWSCredentialsProvider = " + credentialName1,
"workerId = 123",
"failoverTimeMillis = 100"
},
'\n');
getConfiguration(test);
}
@Test
public void testWithMissingApplicationName() {
thrown.expect(NullPointerException.class);
thrown.expectMessage("Application name is required");
@Test(expected = IllegalArgumentException.class)
public void testWithEmptyStreamNameAndMissingStreamArn() {
String test = StringUtils.join(
new String[] {
"applicationName = b",
"AWSCredentialsProvider = " + credentialName1,
"workerId = 123",
"failoverTimeMillis = 100",
"streamName = ",
"streamArn = "
},
'\n');
getConfiguration(test);
}
String test = StringUtils.join(new String[] { "streamName = a", "AWSCredentialsProvider = " + credentialName1,
"workerId = 123", "failoverTimeMillis = 100" }, '\n');
InputStream input = new ByteArrayInputStream(test.getBytes());
configurator.getConfiguration(input);
@Test(expected = NullPointerException.class)
public void testWithMissingApplicationName() {
String test = StringUtils.join(
new String[] {
"streamName = a",
"AWSCredentialsProvider = " + credentialName1,
"workerId = 123",
"failoverTimeMillis = 100"
},
'\n');
getConfiguration(test);
}
@Test
public void testWithAWSCredentialsFailed() {
String test = StringUtils.join(
new String[] { "streamName = a", "applicationName = b", "AWSCredentialsProvider = " + credentialName2,
"failoverTimeMillis = 100", "shardSyncIntervalMillis = 500" },
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = " + credentialName2,
"failoverTimeMillis = 100",
"shardSyncIntervalMillis = 500"
},
'\n');
InputStream input = new ByteArrayInputStream(test.getBytes());
MultiLangDaemonConfiguration config = getConfiguration(test);
// separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement
try {
MultiLangDaemonConfiguration config = configurator.getConfiguration(input);
config.getKinesisCredentialsProvider().build(AwsCredentialsProvider.class).resolveCredentials();
config.getKinesisCredentialsProvider()
.build(AwsCredentialsProvider.class)
.resolveCredentials();
fail("expect failure with wrong credentials provider");
} catch (Exception e) {
// succeed
@ -349,59 +460,63 @@ public class KinesisClientLibConfiguratorTest {
// TODO: fix this test
@Test
public void testWithDifferentAWSCredentialsForDynamoDBAndCloudWatch() {
String test = StringUtils.join(new String[] { "streamName = a", "applicationName = b",
"AWSCredentialsProvider = " + credentialNameKinesis,
"AWSCredentialsProviderDynamoDB = " + credentialNameDynamoDB,
"AWSCredentialsProviderCloudWatch = " + credentialNameCloudWatch, "failoverTimeMillis = 100",
"shardSyncIntervalMillis = 500" }, '\n');
InputStream input = new ByteArrayInputStream(test.getBytes());
String test = StringUtils.join(
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = " + credentialNameKinesis,
"AWSCredentialsProviderDynamoDB = " + credentialNameDynamoDB,
"AWSCredentialsProviderCloudWatch = " + credentialNameCloudWatch,
"failoverTimeMillis = 100",
"shardSyncIntervalMillis = 500"
},
'\n');
// separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement
MultiLangDaemonConfiguration config = configurator.getConfiguration(input);
try {
config.getKinesisCredentialsProvider().build(AwsCredentialsProvider.class).resolveCredentials();
} catch (Exception e) {
fail("Kinesis credential providers should not fail.");
}
try {
config.getDynamoDBCredentialsProvider().build(AwsCredentialsProvider.class).resolveCredentials();
} catch (Exception e) {
fail("DynamoDB credential providers should not fail.");
}
try {
config.getCloudWatchCredentialsProvider().build(AwsCredentialsProvider.class).resolveCredentials();
} catch (Exception e) {
fail("CloudWatch credential providers should not fail.");
}
final MultiLangDaemonConfiguration config = getConfiguration(test);
config.getKinesisCredentialsProvider()
.build(AwsCredentialsProvider.class)
.resolveCredentials();
config.getDynamoDBCredentialsProvider()
.build(AwsCredentialsProvider.class)
.resolveCredentials();
config.getCloudWatchCredentialsProvider()
.build(AwsCredentialsProvider.class)
.resolveCredentials();
}
// TODO: fix this test
@Test
public void testWithDifferentAWSCredentialsForDynamoDBAndCloudWatchFailed() {
String test = StringUtils.join(new String[] { "streamName = a", "applicationName = b",
"AWSCredentialsProvider = " + credentialNameKinesis,
"AWSCredentialsProviderDynamoDB = " + credentialName2,
"AWSCredentialsProviderCloudWatch = " + credentialName2, "failoverTimeMillis = 100",
"shardSyncIntervalMillis = 500" }, '\n');
InputStream input = new ByteArrayInputStream(test.getBytes());
String test = StringUtils.join(
new String[] {
"streamName = a",
"applicationName = b",
"AWSCredentialsProvider = " + credentialNameKinesis,
"AWSCredentialsProviderDynamoDB = " + credentialName2,
"AWSCredentialsProviderCloudWatch = " + credentialName2,
"failoverTimeMillis = 100",
"shardSyncIntervalMillis = 500"
},
'\n');
// separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement
// separate input stream with getConfiguration to explicitly catch exception from the getConfiguration statement
MultiLangDaemonConfiguration config = configurator.getConfiguration(input);
final MultiLangDaemonConfiguration config = getConfiguration(test);
config.getKinesisCredentialsProvider()
.build(AwsCredentialsProvider.class)
.resolveCredentials();
try {
config.getKinesisCredentialsProvider().build(AwsCredentialsProvider.class).resolveCredentials();
} catch (Exception e) {
fail("Kinesis credential providers should not fail.");
}
try {
config.getDynamoDBCredentialsProvider().build(AwsCredentialsProvider.class).resolveCredentials();
config.getDynamoDBCredentialsProvider()
.build(AwsCredentialsProvider.class)
.resolveCredentials();
fail("DynamoDB credential providers should fail.");
} catch (Exception e) {
// succeed
}
try {
config.getCloudWatchCredentialsProvider().build(AwsCredentialsProvider.class).resolveCredentials();
config.getCloudWatchCredentialsProvider()
.build(AwsCredentialsProvider.class)
.resolveCredentials();
fail("CloudWatch credential providers should fail.");
} catch (Exception e) {
// succeed
@ -419,9 +534,7 @@ public class KinesisClientLibConfiguratorTest {
}
@Override
public void refresh() {
}
public void refresh() {}
}
/**
@ -435,9 +548,7 @@ public class KinesisClientLibConfiguratorTest {
}
@Override
public void refresh() {
}
public void refresh() {}
}
/**
@ -451,9 +562,7 @@ public class KinesisClientLibConfiguratorTest {
}
@Override
public void refresh() {
}
public void refresh() {}
}
/**
@ -467,9 +576,7 @@ public class KinesisClientLibConfiguratorTest {
}
@Override
public void refresh() {
}
public void refresh() {}
}
/**
@ -483,14 +590,11 @@ public class KinesisClientLibConfiguratorTest {
}
@Override
public void refresh() {
}
public void refresh() {}
}
private MultiLangDaemonConfiguration getConfiguration(String configString) {
InputStream input = new ByteArrayInputStream(configString.getBytes());
MultiLangDaemonConfiguration config = configurator.getConfiguration(input);
return config;
return configurator.getConfiguration(input);
}
}

152
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/MultiLangDaemonConfigurationTest.java
View file

@ -15,10 +15,6 @@
package software.amazon.kinesis.multilang.config;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.junit.Assert.assertThat;
import org.apache.commons.beanutils.BeanUtilsBean;
import org.apache.commons.beanutils.ConvertUtilsBean;
import org.junit.After;
@ -29,12 +25,18 @@ import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
import software.amazon.kinesis.processor.ShardRecordProcessorFactory;
import software.amazon.kinesis.retrieval.fanout.FanOutConfig;
import software.amazon.kinesis.retrieval.polling.PollingConfig;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
@RunWith(MockitoJUnitRunner.class)
public class MultiLangDaemonConfigurationTest {
@ -67,7 +69,6 @@ public class MultiLangDaemonConfigurationTest {
}
}
public MultiLangDaemonConfiguration baseConfiguration() {
MultiLangDaemonConfiguration configuration = new MultiLangDaemonConfiguration(utilsBean, convertUtilsBean);
configuration.setApplicationName("Test");
@ -82,33 +83,98 @@ public class MultiLangDaemonConfigurationTest {
MultiLangDaemonConfiguration configuration = baseConfiguration();
configuration.setMaxLeasesForWorker(10);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration
.resolvedConfiguration(shardRecordProcessorFactory);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
configuration.resolvedConfiguration(shardRecordProcessorFactory);
assertThat(resolvedConfiguration.leaseManagementConfig.maxLeasesForWorker(), equalTo(10));
}
@Test
public void testSetEnablePriorityLeaseAssignment() {
MultiLangDaemonConfiguration configuration = baseConfiguration();
configuration.setEnablePriorityLeaseAssignment(false);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
configuration.resolvedConfiguration(shardRecordProcessorFactory);
assertThat(resolvedConfiguration.leaseManagementConfig.enablePriorityLeaseAssignment(), equalTo(false));
}
@Test
public void testSetLeaseTableDeletionProtectionEnabledToTrue() {
MultiLangDaemonConfiguration configuration = baseConfiguration();
configuration.setLeaseTableDeletionProtectionEnabled(true);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
configuration.resolvedConfiguration(shardRecordProcessorFactory);
assertTrue(resolvedConfiguration.leaseManagementConfig.leaseTableDeletionProtectionEnabled());
}
@Test
public void testSetLeaseTablePitrEnabledToTrue() {
MultiLangDaemonConfiguration configuration = baseConfiguration();
configuration.setLeaseTablePitrEnabled(true);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
configuration.resolvedConfiguration(shardRecordProcessorFactory);
assertTrue(resolvedConfiguration.leaseManagementConfig.leaseTablePitrEnabled());
}
@Test
public void testSetLeaseTableDeletionProtectionEnabledToFalse() {
MultiLangDaemonConfiguration configuration = baseConfiguration();
configuration.setLeaseTableDeletionProtectionEnabled(false);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
configuration.resolvedConfiguration(shardRecordProcessorFactory);
assertFalse(resolvedConfiguration.leaseManagementConfig.leaseTableDeletionProtectionEnabled());
}
@Test
public void testSetLeaseTablePitrEnabledToFalse() {
MultiLangDaemonConfiguration configuration = baseConfiguration();
configuration.setLeaseTablePitrEnabled(false);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
configuration.resolvedConfiguration(shardRecordProcessorFactory);
assertFalse(resolvedConfiguration.leaseManagementConfig.leaseTablePitrEnabled());
}
@Test
public void testDefaultRetrievalConfig() {
MultiLangDaemonConfiguration configuration = baseConfiguration();
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration
.resolvedConfiguration(shardRecordProcessorFactory);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
configuration.resolvedConfiguration(shardRecordProcessorFactory);
assertThat(resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(),
instanceOf(FanOutConfig.class));
assertThat(
resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), instanceOf(FanOutConfig.class));
}
@Test
public void testDefaultRetrievalConfigWithPollingConfigSet() {
MultiLangDaemonConfiguration configuration = baseConfiguration();
configuration.setMaxRecords(10);
configuration.setIdleTimeBetweenReadsInMillis(60000);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration
.resolvedConfiguration(shardRecordProcessorFactory);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
configuration.resolvedConfiguration(shardRecordProcessorFactory);
assertThat(resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(),
instanceOf(PollingConfig.class));
assertThat(
resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), instanceOf(PollingConfig.class));
assertEquals(
10,
((PollingConfig) resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig()).maxRecords());
assertEquals(
60000,
((PollingConfig) resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig())
.idleTimeBetweenReadsInMillis());
assertTrue(((PollingConfig) resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig())
.usePollingConfigIdleTimeValue());
}
@Test
@ -116,11 +182,11 @@ public class MultiLangDaemonConfigurationTest {
MultiLangDaemonConfiguration configuration = baseConfiguration();
configuration.setRetrievalMode(RetrievalMode.FANOUT);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration
.resolvedConfiguration(shardRecordProcessorFactory);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
configuration.resolvedConfiguration(shardRecordProcessorFactory);
assertThat(resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(),
instanceOf(FanOutConfig.class));
assertThat(
resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), instanceOf(FanOutConfig.class));
}
@Test
@ -128,37 +194,39 @@ public class MultiLangDaemonConfigurationTest {
MultiLangDaemonConfiguration configuration = baseConfiguration();
configuration.setRetrievalMode(RetrievalMode.POLLING);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration
.resolvedConfiguration(shardRecordProcessorFactory);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
configuration.resolvedConfiguration(shardRecordProcessorFactory);
assertThat(resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(),
instanceOf(PollingConfig.class));
assertThat(
resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), instanceOf(PollingConfig.class));
}
@Test
public void testRetrievalModeSetForPollingString() throws Exception {
MultiLangDaemonConfiguration configuration = baseConfiguration();
utilsBean.setProperty(configuration, "retrievalMode", RetrievalMode.POLLING.name().toLowerCase());
utilsBean.setProperty(
configuration, "retrievalMode", RetrievalMode.POLLING.name().toLowerCase());
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration
.resolvedConfiguration(shardRecordProcessorFactory);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
configuration.resolvedConfiguration(shardRecordProcessorFactory);
assertThat(resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(),
instanceOf(PollingConfig.class));
assertThat(
resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), instanceOf(PollingConfig.class));
}
@Test
public void testRetrievalModeSetForFanoutString() throws Exception {
MultiLangDaemonConfiguration configuration = baseConfiguration();
utilsBean.setProperty(configuration, "retrievalMode", RetrievalMode.FANOUT.name().toLowerCase());
utilsBean.setProperty(
configuration, "retrievalMode", RetrievalMode.FANOUT.name().toLowerCase());
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration
.resolvedConfiguration(shardRecordProcessorFactory);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
configuration.resolvedConfiguration(shardRecordProcessorFactory);
assertThat(resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(),
instanceOf(FanOutConfig.class));
assertThat(
resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), instanceOf(FanOutConfig.class));
}
@Test
@ -175,7 +243,7 @@ public class MultiLangDaemonConfigurationTest {
// TODO : Enable this test once https://github.com/awslabs/amazon-kinesis-client/issues/692 is resolved
public void testmetricsEnabledDimensions() {
MultiLangDaemonConfiguration configuration = baseConfiguration();
configuration.setMetricsEnabledDimensions(new String[]{"Operation"});
configuration.setMetricsEnabledDimensions(new String[] {"Operation"});
configuration.resolvedConfiguration(shardRecordProcessorFactory);
}
@ -188,14 +256,14 @@ public class MultiLangDaemonConfigurationTest {
configuration.setRetrievalMode(RetrievalMode.FANOUT);
configuration.getFanoutConfig().setConsumerArn(consumerArn);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration = configuration
.resolvedConfiguration(shardRecordProcessorFactory);
MultiLangDaemonConfiguration.ResolvedConfiguration resolvedConfiguration =
configuration.resolvedConfiguration(shardRecordProcessorFactory);
assertThat(resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(),
instanceOf(FanOutConfig.class));
FanOutConfig fanOutConfig = (FanOutConfig) resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig();
assertThat(
resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig(), instanceOf(FanOutConfig.class));
FanOutConfig fanOutConfig =
(FanOutConfig) resolvedConfiguration.getRetrievalConfig().retrievalSpecificConfig();
assertThat(fanOutConfig.consumerArn(), equalTo(consumerArn));
}
}
}

22
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/config/PollingConfigBeanTest.java
View file

@ -15,6 +15,8 @@
package software.amazon.kinesis.multilang.config;
import java.util.Optional;
import org.apache.commons.beanutils.BeanUtilsBean;
import org.apache.commons.beanutils.ConvertUtilsBean;
import org.junit.Test;
@ -24,8 +26,6 @@ import org.mockito.runners.MockitoJUnitRunner;
import software.amazon.awssdk.services.kinesis.KinesisAsyncClient;
import software.amazon.kinesis.retrieval.polling.PollingConfig;
import java.util.Optional;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.junit.Assert.assertThat;
@ -46,17 +46,23 @@ public class PollingConfigBeanTest {
ConvertUtilsBean convertUtilsBean = new ConvertUtilsBean();
BeanUtilsBean utilsBean = new BeanUtilsBean(convertUtilsBean);
MultiLangDaemonConfiguration multiLangDaemonConfiguration = new MultiLangDaemonConfiguration(utilsBean, convertUtilsBean);
MultiLangDaemonConfiguration multiLangDaemonConfiguration =
new MultiLangDaemonConfiguration(utilsBean, convertUtilsBean);
multiLangDaemonConfiguration.setStreamName("test-stream");
PollingConfig pollingConfig = pollingConfigBean.build(kinesisAsyncClient, multiLangDaemonConfiguration);
assertThat(pollingConfig.kinesisClient(), equalTo(kinesisAsyncClient));
assertThat(pollingConfig.streamName(), equalTo(multiLangDaemonConfiguration.getStreamName()));
assertThat(pollingConfig.idleTimeBetweenReadsInMillis(), equalTo(pollingConfigBean.getIdleTimeBetweenReadsInMillis()));
assertThat(pollingConfig.maxGetRecordsThreadPool(), equalTo(Optional.of(pollingConfigBean.getMaxGetRecordsThreadPool())));
assertThat(
pollingConfig.idleTimeBetweenReadsInMillis(),
equalTo(pollingConfigBean.getIdleTimeBetweenReadsInMillis()));
assertThat(
pollingConfig.maxGetRecordsThreadPool(),
equalTo(Optional.of(pollingConfigBean.getMaxGetRecordsThreadPool())));
assertThat(pollingConfig.maxRecords(), equalTo(pollingConfigBean.getMaxRecords()));
assertThat(pollingConfig.retryGetRecordsInSeconds(), equalTo(Optional.of(pollingConfigBean.getRetryGetRecordsInSeconds())));
assertThat(
pollingConfig.retryGetRecordsInSeconds(),
equalTo(Optional.of(pollingConfigBean.getRetryGetRecordsInSeconds())));
}
}
}

50
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/messages/JsonFriendlyRecordTest.java
View file

@ -15,11 +15,6 @@
package software.amazon.kinesis.multilang.messages;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.nullValue;
import static org.hamcrest.CoreMatchers.sameInstance;
import static org.junit.Assert.assertThat;
import java.nio.ByteBuffer;
import java.time.Instant;
import java.util.Arrays;
@ -31,9 +26,13 @@ import org.hamcrest.Description;
import org.hamcrest.Matcher;
import org.hamcrest.TypeSafeDiagnosingMatcher;
import org.junit.Test;
import software.amazon.kinesis.retrieval.KinesisClientRecord;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.nullValue;
import static org.hamcrest.CoreMatchers.sameInstance;
import static org.junit.Assert.assertThat;
public class JsonFriendlyRecordTest {
private KinesisClientRecord kinesisClientRecord;
@ -48,7 +47,7 @@ public class JsonFriendlyRecordTest {
@Test
public void testRecordHandlesNoByteArrayBuffer() {
byte[] expected = new byte[] { 1, 2, 3, 4 };
byte[] expected = new byte[] {1, 2, 3, 4};
ByteBuffer expectedBuffer = ByteBuffer.allocateDirect(expected.length);
@ -64,7 +63,7 @@ public class JsonFriendlyRecordTest {
@Test
public void testRecordHandlesArrayByteBuffer() {
ByteBuffer expected = ByteBuffer.wrap(new byte[] { 1, 2, 3, 4 });
ByteBuffer expected = ByteBuffer.wrap(new byte[] {1, 2, 3, 4});
kinesisClientRecord = defaultRecord().data(expected).build();
JsonFriendlyRecord jsonFriendlyRecord = JsonFriendlyRecord.fromKinesisClientRecord(kinesisClientRecord);
@ -82,14 +81,15 @@ public class JsonFriendlyRecordTest {
private RecordMatcher(KinesisClientRecord expected) {
this.matchers = Arrays.asList(
new FieldMatcher<>("approximateArrivalTimestamp",
new FieldMatcher<>(
"approximateArrivalTimestamp",
equalTo(expected.approximateArrivalTimestamp().toEpochMilli()),
JsonFriendlyRecord::getApproximateArrivalTimestamp),
new FieldMatcher<>("partitionKey", expected::partitionKey, JsonFriendlyRecord::getPartitionKey),
new FieldMatcher<>("sequenceNumber", expected::sequenceNumber,
JsonFriendlyRecord::getSequenceNumber),
new FieldMatcher<>("subSequenceNumber", expected::subSequenceNumber,
JsonFriendlyRecord::getSubSequenceNumber),
new FieldMatcher<>(
"sequenceNumber", expected::sequenceNumber, JsonFriendlyRecord::getSequenceNumber),
new FieldMatcher<>(
"subSequenceNumber", expected::subSequenceNumber, JsonFriendlyRecord::getSubSequenceNumber),
new FieldMatcher<>("data", dataEquivalentTo(expected.data()), JsonFriendlyRecord::getData));
this.expected = expected;
@ -97,13 +97,16 @@ public class JsonFriendlyRecordTest {
@Override
protected boolean matchesSafely(JsonFriendlyRecord item, Description mismatchDescription) {
return matchers.stream().map(m -> {
if (!m.matches(item)) {
m.describeMismatch(item, mismatchDescription);
return false;
}
return true;
}).reduce((l, r) -> l && r).orElse(true);
return matchers.stream()
.map(m -> {
if (!m.matches(item)) {
m.describeMismatch(item, mismatchDescription);
return false;
}
return true;
})
.reduce((l, r) -> l && r)
.orElse(true);
}
@Override
@ -160,8 +163,9 @@ public class JsonFriendlyRecordTest {
}
private KinesisClientRecord.KinesisClientRecordBuilder defaultRecord() {
return KinesisClientRecord.builder().partitionKey("test-partition").sequenceNumber("123")
return KinesisClientRecord.builder()
.partitionKey("test-partition")
.sequenceNumber("123")
.approximateArrivalTimestamp(Instant.now());
}
}
}

68
amazon-kinesis-client-multilang/src/test/java/software/amazon/kinesis/multilang/messages/MessageTest.java
View file

@ -17,73 +17,65 @@ package software.amazon.kinesis.multilang.messages;
import java.nio.ByteBuffer;
import java.util.Collections;
import org.junit.Assert;
import org.junit.Test;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.Assert;
import org.junit.Test;
import software.amazon.kinesis.lifecycle.ShutdownReason;
import software.amazon.kinesis.lifecycle.events.InitializationInput;
import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput;
import software.amazon.kinesis.lifecycle.ShutdownReason;
import software.amazon.kinesis.multilang.messages.CheckpointMessage;
import software.amazon.kinesis.multilang.messages.InitializeMessage;
import software.amazon.kinesis.multilang.messages.Message;
import software.amazon.kinesis.multilang.messages.ProcessRecordsMessage;
import software.amazon.kinesis.multilang.messages.ShutdownMessage;
import software.amazon.kinesis.multilang.messages.ShutdownRequestedMessage;
import software.amazon.kinesis.multilang.messages.StatusMessage;
import software.amazon.kinesis.retrieval.KinesisClientRecord;
public class MessageTest {
@Test
public void toStringTest() {
Message[] messages = new Message[]{
new CheckpointMessage("1234567890", 0L, null),
new InitializeMessage(InitializationInput.builder().shardId("shard-123").build()),
new ProcessRecordsMessage(ProcessRecordsInput.builder()
.records(Collections.singletonList(
KinesisClientRecord.builder()
.data(ByteBuffer.wrap("cat".getBytes()))
.partitionKey("cat")
.sequenceNumber("555")
.build()))
.build()),
new ShutdownMessage(ShutdownReason.LEASE_LOST),
new StatusMessage("processRecords"),
new InitializeMessage(),
new ProcessRecordsMessage(),
new ShutdownRequestedMessage(),
new LeaseLostMessage(),
new ShardEndedMessage()
Message[] messages = new Message[] {
new CheckpointMessage("1234567890", 0L, null),
new InitializeMessage(
InitializationInput.builder().shardId("shard-123").build()),
new ProcessRecordsMessage(ProcessRecordsInput.builder()
.records(Collections.singletonList(KinesisClientRecord.builder()
.data(ByteBuffer.wrap("cat".getBytes()))
.partitionKey("cat")
.sequenceNumber("555")
.build()))
.build()),
new ShutdownMessage(ShutdownReason.LEASE_LOST),
new StatusMessage("processRecords"),
new InitializeMessage(),
new ProcessRecordsMessage(),
new ShutdownRequestedMessage(),
new LeaseLostMessage(),
new ShardEndedMessage(),
};
// TODO: fix this
// TODO: fix this
for (int i = 0; i < messages.length; i++) {
System.out.println(messages[i].toString());
Assert.assertTrue("Each message should contain the action field", messages[i].toString().contains("action"));
Assert.assertTrue(
"Each message should contain the action field",
messages[i].toString().contains("action"));
}
// Hit this constructor
KinesisClientRecord defaultJsonFriendlyRecord = KinesisClientRecord.builder().build();
KinesisClientRecord defaultJsonFriendlyRecord =
KinesisClientRecord.builder().build();
Assert.assertNull(defaultJsonFriendlyRecord.partitionKey());
Assert.assertNull(defaultJsonFriendlyRecord.data());
Assert.assertNull(defaultJsonFriendlyRecord.sequenceNumber());
Assert.assertNull(new ShutdownMessage(null).getReason());
// Hit the bad object mapping path
Message withBadMapper = new Message() {
}.withObjectMapper(new ObjectMapper() {
Message withBadMapper = new Message() {}.withObjectMapper(new ObjectMapper() {
/**
*
*
*/
private static final long serialVersionUID = 1L;
@Override
public String writeValueAsString(Object m) throws JsonProcessingException {
throw new JsonProcessingException(new Throwable()) {
};
throw new JsonProcessingException(new Throwable()) {};
}
});
String s = withBadMapper.toString();

93
amazon-kinesis-client-multilang/src/test/resources/multilang.properties Normal file
View file

@ -0,0 +1,93 @@
# The script that abides by the multi-language protocol. This script will
# be executed by the MultiLangDaemon, which will communicate with this script
# over STDIN and STDOUT according to the multi-language protocol.
executableName = sample_kclpy_app.py
# The Stream arn: arn:aws:kinesis:<region>:<account id>:stream/<stream name>
# Important: streamArn takes precedence over streamName if both are set
streamArn = arn:aws:kinesis:us-east-5:000000000000:stream/kclpysample
# The name of an Amazon Kinesis stream to process.
# Important: streamArn takes precedence over streamName if both are set
streamName = kclpysample
# Used by the KCL as the name of this application. Will be used as the name
# of an Amazon DynamoDB table which will store the lease and checkpoint
# information for workers with this application name
applicationName = MultiLangTest
# Users can change the credentials provider the KCL will use to retrieve credentials.
# The DefaultAWSCredentialsProviderChain checks several other providers, which is
# described here:
# http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html
AWSCredentialsProvider = DefaultAWSCredentialsProviderChain
# Appended to the user agent of the KCL. Does not impact the functionality of the
# KCL in any other way.
processingLanguage = python/3.8
# Valid options at TRIM_HORIZON or LATEST.
# See http://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetShardIterator.html#API_GetShardIterator_RequestSyntax
initialPositionInStream = TRIM_HORIZON
# To specify an initial timestamp from which to start processing records, please specify timestamp value for 'initiatPositionInStreamExtended',
# and uncomment below line with right timestamp value.
# See more from 'Timestamp' under http://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetShardIterator.html#API_GetShardIterator_RequestSyntax
#initialPositionInStreamExtended = 1636609142
# The following properties are also available for configuring the KCL Worker that is created
# by the MultiLangDaemon.
# The KCL defaults to us-east-1
regionName = us-east-1
# Fail over time in milliseconds. A worker which does not renew it's lease within this time interval
# will be regarded as having problems and it's shards will be assigned to other workers.
# For applications that have a large number of shards, this msy be set to a higher number to reduce
# the number of DynamoDB IOPS required for tracking leases
failoverTimeMillis = 10000
# A worker id that uniquely identifies this worker among all workers using the same applicationName
# If this isn't provided a MultiLangDaemon instance will assign a unique workerId to itself.
workerId = "workerId"
# Shard sync interval in milliseconds - e.g. wait for this long between shard sync tasks.
shardSyncIntervalMillis = 60000
# Max records to fetch from Kinesis in a single GetRecords call.
maxRecords = 10000
# Idle time between record reads in milliseconds.
idleTimeBetweenReadsInMillis = 1000
# Enables applications flush/checkpoint (if they have some data "in progress", but don't get new data for while)
callProcessRecordsEvenForEmptyRecordList = false
# Interval in milliseconds between polling to check for parent shard completion.
# Polling frequently will take up more DynamoDB IOPS (when there are leases for shards waiting on
# completion of parent shards).
parentShardPollIntervalMillis = 10000
# Cleanup leases upon shards completion (don't wait until they expire in Kinesis).
# Keeping leases takes some tracking/resources (e.g. they need to be renewed, assigned), so by default we try
# to delete the ones we don't need any longer.
cleanupLeasesUponShardCompletion = true
# Backoff time in milliseconds for Amazon Kinesis Client Library tasks (in the event of failures).
taskBackoffTimeMillis = 500
# Buffer metrics for at most this long before publishing to CloudWatch.
metricsBufferTimeMillis = 10000
# Buffer at most this many metrics before publishing to CloudWatch.
metricsMaxQueueSize = 10000
# KCL will validate client provided sequence numbers with a call to Amazon Kinesis before checkpointing for calls
# to RecordProcessorCheckpointer#checkpoint(String) by default.
validateSequenceNumberBeforeCheckpointing = true
# The maximum number of active threads for the MultiLangDaemon to permit.
# If a value is provided then a FixedThreadPool is used with the maximum
# active threads set to the provided value. If a non-positive integer or no
# value is provided a CachedThreadPool is used.
maxActiveThreads = -1

99
amazon-kinesis-client/pom.xml
View file

@ -1,4 +1,5 @@
<!--
/*
* Copyright 2019 Amazon.com, Inc. or its affiliates.
* Licensed under the Apache License, Version 2.0 (the
@ -22,7 +23,7 @@
<parent>
<groupId>software.amazon.kinesis</groupId>
<artifactId>amazon-kinesis-client-pom</artifactId>
<version>${revision}</version>
<version>2.6.1-SNAPSHOT</version>
</parent>
<artifactId>amazon-kinesis-client</artifactId>
@ -47,11 +48,13 @@
</licenses>
<properties>
<protobuf.version>4.27.0</protobuf.version>
<sqlite4java.version>1.0.392</sqlite4java.version>
<sqlite4java.native>libsqlite4java</sqlite4java.native>
<sqlite4java.libpath>${project.build.directory}/test-lib</sqlite4java.libpath>
<slf4j.version>2.0.6</slf4j.version>
<gsr.version>1.1.13</gsr.version>
<slf4j.version>2.0.13</slf4j.version>
<gsr.version>1.1.19</gsr.version>
<skipITs>true</skipITs>
</properties>
<dependencies>
@ -88,17 +91,17 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>31.1-jre</version>
<version>32.1.1-jre</version>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>3.21.12</version>
<version>${protobuf.version}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.12.0</version>
<version>3.14.0</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
@ -109,7 +112,7 @@
<dependency>
<groupId>io.reactivex.rxjava3</groupId>
<artifactId>rxjava</artifactId>
<version>3.1.6</version>
<version>3.1.8</version>
</dependency>
<dependency>
@ -152,7 +155,7 @@
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>1.3.0</version>
<version>1.3.14</version>
<scope>test</scope>
</dependency>
@ -178,15 +181,21 @@
</developers>
<build>
<extensions>
<extension>
<groupId>kr.motd.maven</groupId>
<artifactId>os-maven-plugin</artifactId>
<version>1.6.0</version>
</extension>
</extensions>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.1</version>
<version>3.13.0</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
<release>8</release>
<encoding>UTF-8</encoding>
</configuration>
</plugin>
@ -194,26 +203,41 @@
</pluginManagement>
<plugins>
<plugin>
<groupId>org.xolstice.maven.plugins</groupId>
<artifactId>protobuf-maven-plugin</artifactId>
<version>0.6.1</version>
<executions>
<execution>
<goals>
<goal>compile</goal>
</goals>
</execution>
</executions>
<configuration>
<protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.22.2</version>
<version>3.2.5</version>
<configuration>
<skipTests>${skip.ut}</skipTests>
<skipITs>${skipITs}</skipITs>
<excludes>
<exclude>**/*IntegrationTest.java</exclude>
</excludes>
<systemProperties>
<property>
<name>sqlite4java.library.path</name>
<value>${sqlite4java.libpath}</value>
</property>
</systemProperties>
<systemPropertyVariables>
<sqlite4java.library.path>${sqlite4java.libpath}</sqlite4java.library.path>
<awsProfile>${awsProfile}</awsProfile>
</systemPropertyVariables>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<version>2.22.2</version>
<version>3.2.5</version>
<configuration>
<includes>
<include>**/*IntegrationTest.java</include>
@ -299,7 +323,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>3.3.1</version>
<version>3.7.0</version>
<configuration>
<excludePackageNames>com.amazonaws.services.kinesis.producer.protobuf</excludePackageNames>
</configuration>
@ -343,7 +367,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
<version>3.3.0</version>
<version>3.3.1</version>
<executions>
<execution>
<id>copy-dist</id>
@ -365,6 +389,39 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>com.diffplug.spotless</groupId>
<artifactId>spotless-maven-plugin</artifactId>
<version>2.30.0</version> <!--last version to support java 8-->
<configuration>
<java>
<palantirJavaFormat />
<importOrder>
<order>java,,\#</order>
</importOrder>
</java>
</configuration>
<executions>
<execution>
<goals>
<goal>check</goal>
</goals>
<phase>compile</phase>
</execution>
</executions>
</plugin>
<plugin>
<groupId>com.salesforce.servicelibs</groupId>
<artifactId>proto-backwards-compatibility</artifactId>
<version>1.0.7</version>
<executions>
<execution>
<goals>
<goal>backwards-compatibility-check</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>

3
amazon-kinesis-client/src/main/java/software/amazon/kinesis/annotations/KinesisClientInternalApi.java
View file

@ -22,5 +22,4 @@ import java.lang.annotation.RetentionPolicy;
* Any class/method/variable marked with this annotation is subject to breaking changes between minor releases.
*/
@Retention(RetentionPolicy.CLASS)
public @interface KinesisClientInternalApi {
}
public @interface KinesisClientInternalApi {}

5
amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/Checkpoint.java
View file

@ -40,7 +40,10 @@ public class Checkpoint {
* @param pendingCheckpoint the pending checkpoint sequence number - can be null.
* @param pendingCheckpointState the pending checkpoint state - can be null.
*/
public Checkpoint(final ExtendedSequenceNumber checkpoint, final ExtendedSequenceNumber pendingCheckpoint, byte[] pendingCheckpointState) {
public Checkpoint(
final ExtendedSequenceNumber checkpoint,
final ExtendedSequenceNumber pendingCheckpoint,
byte[] pendingCheckpointState) {
if (checkpoint == null || checkpoint.sequenceNumber().isEmpty()) {
throw new IllegalArgumentException("Checkpoint cannot be null or empty");
}

1
amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/CheckpointConfig.java
View file

@ -15,7 +15,6 @@
package software.amazon.kinesis.checkpoint;
import lombok.Data;
import lombok.experimental.Accessors;
import software.amazon.kinesis.checkpoint.dynamodb.DynamoDBCheckpointFactory;

4
amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/DoesNothingPreparedCheckpointer.java
View file

@ -60,9 +60,7 @@ public class DoesNothingPreparedCheckpointer implements PreparedCheckpointer {
@Override
public void checkpoint()
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException,
IllegalArgumentException {
IllegalArgumentException {
// This method does nothing
}
}

16
amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/SequenceNumberValidator.java
View file

@ -20,10 +20,9 @@ import java.util.Collections;
import java.util.List;
import java.util.Optional;
import org.apache.commons.lang3.StringUtils;
import lombok.Data;
import lombok.experimental.Accessors;
import org.apache.commons.lang3.StringUtils;
/**
* This supports extracting the shardId from a sequence number.
@ -98,11 +97,15 @@ public class SequenceNumberValidator {
}
}
private static final List<SequenceNumberReader> SEQUENCE_NUMBER_READERS = Collections
.singletonList(new V2SequenceNumberReader());
private static final List<SequenceNumberReader> SEQUENCE_NUMBER_READERS =
Collections.singletonList(new V2SequenceNumberReader());
private Optional<SequenceNumberComponents> retrieveComponentsFor(String sequenceNumber) {
return SEQUENCE_NUMBER_READERS.stream().map(r -> r.read(sequenceNumber)).filter(Optional::isPresent).map(Optional::get).findFirst();
return SEQUENCE_NUMBER_READERS.stream()
.map(r -> r.read(sequenceNumber))
.filter(Optional::isPresent)
.map(Optional::get)
.findFirst();
}
/**
@ -118,7 +121,7 @@ public class SequenceNumberValidator {
* </ul>
* </strong>
* </p>
*
*
* @param sequenceNumber
* the sequence number to extract the version from
* @return an Optional containing the version if a compatible sequence number reader can be found, an empty Optional
@ -184,5 +187,4 @@ public class SequenceNumberValidator {
public Optional<Boolean> validateSequenceNumberForShard(String sequenceNumber, String shardId) {
return shardIdFor(sequenceNumber).map(s -> StringUtils.equalsIgnoreCase(s, shardId));
}
}

10
amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/ShardPreparedCheckpointer.java
View file

@ -38,8 +38,8 @@ public class ShardPreparedCheckpointer implements PreparedCheckpointer {
* @param pendingCheckpointSequenceNumber sequence number to checkpoint at
* @param checkpointer checkpointer to use
*/
public ShardPreparedCheckpointer(ExtendedSequenceNumber pendingCheckpointSequenceNumber,
RecordProcessorCheckpointer checkpointer) {
public ShardPreparedCheckpointer(
ExtendedSequenceNumber pendingCheckpointSequenceNumber, RecordProcessorCheckpointer checkpointer) {
this.pendingCheckpointSequenceNumber = pendingCheckpointSequenceNumber;
this.checkpointer = checkpointer;
}
@ -58,8 +58,8 @@ public class ShardPreparedCheckpointer implements PreparedCheckpointer {
@Override
public void checkpoint()
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException,
IllegalArgumentException {
checkpointer.checkpoint(pendingCheckpointSequenceNumber.sequenceNumber(),
pendingCheckpointSequenceNumber.subSequenceNumber());
IllegalArgumentException {
checkpointer.checkpoint(
pendingCheckpointSequenceNumber.sequenceNumber(), pendingCheckpointSequenceNumber.subSequenceNumber());
}
}

114
amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/ShardRecordProcessorCheckpointer.java
View file

@ -41,16 +41,22 @@ import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber;
public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpointer {
@NonNull
private final ShardInfo shardInfo;
@NonNull
@Getter @Accessors(fluent = true)
@Getter
@Accessors(fluent = true)
private final Checkpointer checkpointer;
// Set to the last value set via checkpoint().
// Sample use: verify application shutdown() invoked checkpoint() at the end of a shard.
@Getter @Accessors(fluent = true)
@Getter
@Accessors(fluent = true)
private ExtendedSequenceNumber lastCheckpointValue;
@Getter @Accessors(fluent = true)
@Getter
@Accessors(fluent = true)
private ExtendedSequenceNumber largestPermittedCheckpointValue;
private ExtendedSequenceNumber sequenceNumberAtShardEnd;
/**
@ -60,8 +66,11 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
public synchronized void checkpoint()
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
if (log.isDebugEnabled()) {
log.debug("Checkpointing {}, token {} at largest permitted value {}", ShardInfo.getLeaseKey(shardInfo),
shardInfo.concurrencyToken(), this.largestPermittedCheckpointValue);
log.debug(
"Checkpointing {}, token {} at largest permitted value {}",
ShardInfo.getLeaseKey(shardInfo),
shardInfo.concurrencyToken(),
this.largestPermittedCheckpointValue);
}
advancePosition(this.largestPermittedCheckpointValue);
}
@ -71,15 +80,15 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
*/
@Override
public synchronized void checkpoint(Record record)
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException,
IllegalArgumentException {
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException,
IllegalArgumentException {
// TODO: UserRecord Deprecation
if (record == null) {
throw new IllegalArgumentException("Could not checkpoint a null record");
} /* else if (record instanceof UserRecord) {
checkpoint(record.sequenceNumber(), ((UserRecord) record).subSequenceNumber());
} */ else {
checkpoint(record.sequenceNumber(), ((UserRecord) record).subSequenceNumber());
} */ else {
checkpoint(record.sequenceNumber(), 0);
}
}
@ -89,8 +98,8 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
*/
@Override
public synchronized void checkpoint(String sequenceNumber)
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException,
IllegalArgumentException {
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException,
IllegalArgumentException {
checkpoint(sequenceNumber, 0);
}
@ -99,12 +108,12 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
*/
@Override
public synchronized void checkpoint(String sequenceNumber, long subSequenceNumber)
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException,
IllegalArgumentException {
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException,
IllegalArgumentException {
if (subSequenceNumber < 0) {
throw new IllegalArgumentException("Could not checkpoint at invalid, negative subsequence number "
+ subSequenceNumber);
throw new IllegalArgumentException(
"Could not checkpoint at invalid, negative subsequence number " + subSequenceNumber);
}
/*
@ -116,15 +125,18 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
&& newCheckpoint.compareTo(largestPermittedCheckpointValue) <= 0) {
if (log.isDebugEnabled()) {
log.debug("Checkpointing {}, token {} at specific extended sequence number {}", ShardInfo.getLeaseKey(shardInfo),
shardInfo.concurrencyToken(), newCheckpoint);
log.debug(
"Checkpointing {}, token {} at specific extended sequence number {}",
ShardInfo.getLeaseKey(shardInfo),
shardInfo.concurrencyToken(),
newCheckpoint);
}
this.advancePosition(newCheckpoint);
} else {
throw new IllegalArgumentException(String.format(
"Could not checkpoint at extended sequence number %s as it did not fall into acceptable range "
+ "between the last checkpoint %s and the greatest extended sequence number passed to this "
+ "record processor %s",
+ "between the last checkpoint %s and the greatest extended sequence number passed to this "
+ "record processor %s",
newCheckpoint, this.lastCheckpointValue, this.largestPermittedCheckpointValue));
}
}
@ -144,7 +156,8 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
* {@inheritDoc}
*/
@Override
public PreparedCheckpointer prepareCheckpoint(byte[] applicationState) throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
public PreparedCheckpointer prepareCheckpoint(byte[] applicationState)
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
return prepareCheckpoint(largestPermittedCheckpointValue.sequenceNumber(), applicationState);
}
@ -152,15 +165,16 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
* {@inheritDoc}
*/
@Override
public PreparedCheckpointer prepareCheckpoint(Record record, byte[] applicationState) throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
public PreparedCheckpointer prepareCheckpoint(Record record, byte[] applicationState)
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
//
// TODO: UserRecord Deprecation
//
if (record == null) {
throw new IllegalArgumentException("Could not prepare checkpoint a null record");
} /*else if (record instanceof UserRecord) {
return prepareCheckpoint(record.sequenceNumber(), ((UserRecord) record).subSequenceNumber());
} */ else {
return prepareCheckpoint(record.sequenceNumber(), ((UserRecord) record).subSequenceNumber());
} */ else {
return prepareCheckpoint(record.sequenceNumber(), 0, applicationState);
}
}
@ -188,7 +202,8 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
*/
@Override
public PreparedCheckpointer prepareCheckpoint(String sequenceNumber, byte[] applicationState)
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException, IllegalArgumentException {
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException,
IllegalArgumentException {
return prepareCheckpoint(sequenceNumber, 0, applicationState);
}
@ -205,11 +220,13 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
* {@inheritDoc}
*/
@Override
public PreparedCheckpointer prepareCheckpoint(String sequenceNumber, long subSequenceNumber, byte[] applicationState)
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException, IllegalArgumentException {
public PreparedCheckpointer prepareCheckpoint(
String sequenceNumber, long subSequenceNumber, byte[] applicationState)
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException,
IllegalArgumentException {
if (subSequenceNumber < 0) {
throw new IllegalArgumentException("Could not checkpoint at invalid, negative subsequence number "
+ subSequenceNumber);
throw new IllegalArgumentException(
"Could not checkpoint at invalid, negative subsequence number " + subSequenceNumber);
}
/*
@ -221,8 +238,11 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
&& pendingCheckpoint.compareTo(largestPermittedCheckpointValue) <= 0) {
if (log.isDebugEnabled()) {
log.debug("Preparing checkpoint {}, token {} at specific extended sequence number {}",
ShardInfo.getLeaseKey(shardInfo), shardInfo.concurrencyToken(), pendingCheckpoint);
log.debug(
"Preparing checkpoint {}, token {} at specific extended sequence number {}",
ShardInfo.getLeaseKey(shardInfo),
shardInfo.concurrencyToken(),
pendingCheckpoint);
}
return doPrepareCheckpoint(pendingCheckpoint, applicationState);
} else {
@ -256,7 +276,6 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
this.sequenceNumberAtShardEnd = extendedSequenceNumber;
}
/**
* Internal API - has package level access only for testing purposes.
*
@ -268,29 +287,35 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
* @throws InvalidStateException
*/
void advancePosition(String sequenceNumber)
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
advancePosition(new ExtendedSequenceNumber(sequenceNumber));
}
void advancePosition(ExtendedSequenceNumber extendedSequenceNumber)
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
ExtendedSequenceNumber checkpointToRecord = extendedSequenceNumber;
if (sequenceNumberAtShardEnd != null && sequenceNumberAtShardEnd.equals(extendedSequenceNumber)) {
// If we are about to checkpoint the very last sequence number for this shard, we might as well
// just checkpoint at SHARD_END
checkpointToRecord = ExtendedSequenceNumber.SHARD_END;
}
// Don't checkpoint a value we already successfully checkpointed
if (extendedSequenceNumber != null && !extendedSequenceNumber.equals(lastCheckpointValue)) {
try {
if (log.isDebugEnabled()) {
log.debug("Setting {}, token {} checkpoint to {}", ShardInfo.getLeaseKey(shardInfo),
shardInfo.concurrencyToken(), checkpointToRecord);
log.debug(
"Setting {}, token {} checkpoint to {}",
ShardInfo.getLeaseKey(shardInfo),
shardInfo.concurrencyToken(),
checkpointToRecord);
}
checkpointer.setCheckpoint(ShardInfo.getLeaseKey(shardInfo), checkpointToRecord, shardInfo.concurrencyToken());
checkpointer.setCheckpoint(
ShardInfo.getLeaseKey(shardInfo), checkpointToRecord, shardInfo.concurrencyToken());
lastCheckpointValue = checkpointToRecord;
} catch (ThrottlingException | ShutdownException | InvalidStateException
} catch (ThrottlingException
| ShutdownException
| InvalidStateException
| KinesisClientLibDependencyException e) {
throw e;
} catch (KinesisClientLibException e) {
@ -323,7 +348,8 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
* @throws ThrottlingException
* @throws ShutdownException
*/
private PreparedCheckpointer doPrepareCheckpoint(ExtendedSequenceNumber extendedSequenceNumber, byte[] applicationState)
private PreparedCheckpointer doPrepareCheckpoint(
ExtendedSequenceNumber extendedSequenceNumber, byte[] applicationState)
throws KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
ExtendedSequenceNumber newPrepareCheckpoint = extendedSequenceNumber;
@ -341,8 +367,14 @@ public class ShardRecordProcessorCheckpointer implements RecordProcessorCheckpoi
}
try {
checkpointer.prepareCheckpoint(ShardInfo.getLeaseKey(shardInfo), newPrepareCheckpoint, shardInfo.concurrencyToken(), applicationState);
} catch (ThrottlingException | ShutdownException | InvalidStateException
checkpointer.prepareCheckpoint(
ShardInfo.getLeaseKey(shardInfo),
newPrepareCheckpoint,
shardInfo.concurrencyToken(),
applicationState);
} catch (ThrottlingException
| ShutdownException
| InvalidStateException
| KinesisClientLibDependencyException e) {
throw e;
} catch (KinesisClientLibException e) {

5
amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/dynamodb/DynamoDBCheckpointFactory.java
View file

@ -29,9 +29,8 @@ import software.amazon.kinesis.processor.Checkpointer;
@KinesisClientInternalApi
public class DynamoDBCheckpointFactory implements CheckpointFactory {
@Override
public Checkpointer createCheckpointer(final LeaseCoordinator leaseLeaseCoordinator,
final LeaseRefresher leaseRefresher) {
public Checkpointer createCheckpointer(
final LeaseCoordinator leaseLeaseCoordinator, final LeaseRefresher leaseRefresher) {
return new DynamoDBCheckpointer(leaseLeaseCoordinator, leaseRefresher);
}
}

41
amazon-kinesis-client/src/main/java/software/amazon/kinesis/checkpoint/dynamodb/DynamoDBCheckpointer.java
View file

@ -19,7 +19,6 @@ import java.util.Objects;
import java.util.UUID;
import com.google.common.annotations.VisibleForTesting;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@ -48,14 +47,16 @@ import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber;
public class DynamoDBCheckpointer implements Checkpointer {
@NonNull
private final LeaseCoordinator leaseCoordinator;
@NonNull
private final LeaseRefresher leaseRefresher;
private String operation;
@Override
public void setCheckpoint(final String leaseKey, final ExtendedSequenceNumber checkpointValue,
final String concurrencyToken) throws KinesisClientLibException {
public void setCheckpoint(
final String leaseKey, final ExtendedSequenceNumber checkpointValue, final String concurrencyToken)
throws KinesisClientLibException {
try {
boolean wasSuccessful = setCheckpoint(leaseKey, checkpointValue, UUID.fromString(concurrencyToken));
if (!wasSuccessful) {
@ -97,16 +98,22 @@ public class DynamoDBCheckpointer implements Checkpointer {
}
@Override
public void prepareCheckpoint(final String leaseKey, final ExtendedSequenceNumber pendingCheckpoint,
final String concurrencyToken) throws KinesisClientLibException {
public void prepareCheckpoint(
final String leaseKey, final ExtendedSequenceNumber pendingCheckpoint, final String concurrencyToken)
throws KinesisClientLibException {
prepareCheckpoint(leaseKey, pendingCheckpoint, concurrencyToken, null);
}
@Override
public void prepareCheckpoint(String leaseKey, ExtendedSequenceNumber pendingCheckpoint, String concurrencyToken, byte[] pendingCheckpointState) throws KinesisClientLibException {
public void prepareCheckpoint(
String leaseKey,
ExtendedSequenceNumber pendingCheckpoint,
String concurrencyToken,
byte[] pendingCheckpointState)
throws KinesisClientLibException {
try {
boolean wasSuccessful =
prepareCheckpoint(leaseKey, pendingCheckpoint, UUID.fromString(concurrencyToken), pendingCheckpointState);
boolean wasSuccessful = prepareCheckpoint(
leaseKey, pendingCheckpoint, UUID.fromString(concurrencyToken), pendingCheckpointState);
if (!wasSuccessful) {
throw new ShutdownException(
"Can't prepare checkpoint - instance doesn't hold the lease for this shard");
@ -127,8 +134,10 @@ public class DynamoDBCheckpointer implements Checkpointer {
throws DependencyException, InvalidStateException, ProvisionedThroughputException {
Lease lease = leaseCoordinator.getCurrentlyHeldLease(leaseKey);
if (lease == null) {
log.info("Worker {} could not update checkpoint for shard {} because it does not hold the lease",
leaseCoordinator.workerIdentifier(), leaseKey);
log.info(
"Worker {} could not update checkpoint for shard {} because it does not hold the lease",
leaseCoordinator.workerIdentifier(),
leaseKey);
return false;
}
@ -140,12 +149,18 @@ public class DynamoDBCheckpointer implements Checkpointer {
return leaseCoordinator.updateLease(lease, concurrencyToken, operation, leaseKey);
}
boolean prepareCheckpoint(String leaseKey, ExtendedSequenceNumber pendingCheckpoint, UUID concurrencyToken, byte[] pendingCheckpointState)
boolean prepareCheckpoint(
String leaseKey,
ExtendedSequenceNumber pendingCheckpoint,
UUID concurrencyToken,
byte[] pendingCheckpointState)
throws DependencyException, InvalidStateException, ProvisionedThroughputException {
Lease lease = leaseCoordinator.getCurrentlyHeldLease(leaseKey);
if (lease == null) {
log.info("Worker {} could not prepare checkpoint for shard {} because it does not hold the lease",
leaseCoordinator.workerIdentifier(), leaseKey);
log.info(
"Worker {} could not prepare checkpoint for shard {} because it does not hold the lease",
leaseCoordinator.workerIdentifier(),
leaseKey);
return false;
}

32
amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/ArnUtil.java Normal file
View file

@ -0,0 +1,32 @@
package software.amazon.kinesis.common;
import lombok.NonNull;
import software.amazon.awssdk.arns.Arn;
import software.amazon.awssdk.regions.Region;
import software.amazon.kinesis.annotations.KinesisClientInternalApi;
import static software.amazon.awssdk.services.kinesis.KinesisAsyncClient.SERVICE_NAME;
@KinesisClientInternalApi
public final class ArnUtil {
private static final String STREAM_RESOURCE_PREFIX = "stream/";
/**
* Construct a Kinesis stream ARN.
*
* @param region The region the stream exists in.
* @param accountId The account the stream belongs to.
* @param streamName The name of the stream.
* @return The {@link Arn} of the Kinesis stream.
*/
public static Arn constructStreamArn(
@NonNull final Region region, @NonNull final String accountId, @NonNull final String streamName) {
return Arn.builder()
.partition(region.metadata().partition().id())
.service(SERVICE_NAME)
.region(region.id())
.accountId(accountId)
.resource(STREAM_RESOURCE_PREFIX + streamName)
.build();
}
}

2
amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/CommonCalculations.java
View file

@ -15,10 +15,8 @@
package software.amazon.kinesis.common;
public class CommonCalculations {
/**
* Convenience method for calculating renewer intervals in milliseconds.
*

74
amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/ConfigsBuilder.java
View file

@ -19,12 +19,12 @@ import java.util.function.Function;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.NonNull;
import lombok.Setter;
import lombok.ToString;
import org.apache.commons.lang3.StringUtils;
import lombok.NonNull;
import lombok.experimental.Accessors;
import org.apache.commons.lang3.StringUtils;
import software.amazon.awssdk.arns.Arn;
import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient;
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
import software.amazon.awssdk.services.kinesis.KinesisAsyncClient;
@ -34,9 +34,9 @@ import software.amazon.kinesis.coordinator.CoordinatorConfig;
import software.amazon.kinesis.leases.LeaseManagementConfig;
import software.amazon.kinesis.lifecycle.LifecycleConfig;
import software.amazon.kinesis.metrics.MetricsConfig;
import software.amazon.kinesis.processor.MultiStreamTracker;
import software.amazon.kinesis.processor.ProcessorConfig;
import software.amazon.kinesis.processor.ShardRecordProcessorFactory;
import software.amazon.kinesis.processor.MultiStreamTracker;
import software.amazon.kinesis.processor.SingleStreamTracker;
import software.amazon.kinesis.processor.StreamTracker;
import software.amazon.kinesis.retrieval.RetrievalConfig;
@ -44,7 +44,10 @@ import software.amazon.kinesis.retrieval.RetrievalConfig;
/**
* This Builder is useful to create all configurations for the KCL with default values.
*/
@Getter @Setter @ToString @EqualsAndHashCode
@Getter
@Setter
@ToString
@EqualsAndHashCode
@Accessors(fluent = true)
public class ConfigsBuilder {
/**
@ -128,7 +131,7 @@ public class ConfigsBuilder {
}
/**
* Constructor to initialize ConfigsBuilder for a single stream.
* Constructor to initialize ConfigsBuilder for a single stream identified by name.
*
* @param streamName
* @param applicationName
@ -138,11 +141,45 @@ public class ConfigsBuilder {
* @param workerIdentifier
* @param shardRecordProcessorFactory
*/
public ConfigsBuilder(@NonNull String streamName, @NonNull String applicationName,
@NonNull KinesisAsyncClient kinesisClient, @NonNull DynamoDbAsyncClient dynamoDBClient,
@NonNull CloudWatchAsyncClient cloudWatchClient, @NonNull String workerIdentifier,
public ConfigsBuilder(
@NonNull String streamName,
@NonNull String applicationName,
@NonNull KinesisAsyncClient kinesisClient,
@NonNull DynamoDbAsyncClient dynamoDBClient,
@NonNull CloudWatchAsyncClient cloudWatchClient,
@NonNull String workerIdentifier,
@NonNull ShardRecordProcessorFactory shardRecordProcessorFactory) {
this(new SingleStreamTracker(streamName),
this(
new SingleStreamTracker(streamName),
applicationName,
kinesisClient,
dynamoDBClient,
cloudWatchClient,
workerIdentifier,
shardRecordProcessorFactory);
}
/**
* Constructor to initialize ConfigsBuilder for a single stream identified by {@link Arn}.
*
* @param streamArn
* @param applicationName
* @param kinesisClient
* @param dynamoDBClient
* @param cloudWatchClient
* @param workerIdentifier
* @param shardRecordProcessorFactory
*/
public ConfigsBuilder(
@NonNull Arn streamArn,
@NonNull String applicationName,
@NonNull KinesisAsyncClient kinesisClient,
@NonNull DynamoDbAsyncClient dynamoDBClient,
@NonNull CloudWatchAsyncClient cloudWatchClient,
@NonNull String workerIdentifier,
@NonNull ShardRecordProcessorFactory shardRecordProcessorFactory) {
this(
new SingleStreamTracker(streamArn),
applicationName,
kinesisClient,
dynamoDBClient,
@ -162,9 +199,13 @@ public class ConfigsBuilder {
* @param workerIdentifier
* @param shardRecordProcessorFactory
*/
public ConfigsBuilder(@NonNull StreamTracker streamTracker, @NonNull String applicationName,
@NonNull KinesisAsyncClient kinesisClient, @NonNull DynamoDbAsyncClient dynamoDBClient,
@NonNull CloudWatchAsyncClient cloudWatchClient, @NonNull String workerIdentifier,
public ConfigsBuilder(
@NonNull StreamTracker streamTracker,
@NonNull String applicationName,
@NonNull KinesisAsyncClient kinesisClient,
@NonNull DynamoDbAsyncClient dynamoDBClient,
@NonNull CloudWatchAsyncClient cloudWatchClient,
@NonNull String workerIdentifier,
@NonNull ShardRecordProcessorFactory shardRecordProcessorFactory) {
this.applicationName = applicationName;
this.kinesisClient = kinesisClient;
@ -184,8 +225,11 @@ public class ConfigsBuilder {
public void streamTracker(StreamTracker streamTracker) {
this.streamTracker = streamTracker;
this.appStreamTracker = DeprecationUtils.convert(streamTracker,
singleStreamTracker -> singleStreamTracker.streamConfigList().get(0).streamIdentifier().streamName());
this.appStreamTracker = DeprecationUtils.convert(streamTracker, singleStreamTracker -> singleStreamTracker
.streamConfigList()
.get(0)
.streamIdentifier()
.streamName());
}
/**

4
amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/DeprecationUtils.java
View file

@ -39,8 +39,7 @@ public final class DeprecationUtils {
*/
@Deprecated
public static <R> Either<MultiStreamTracker, R> convert(
StreamTracker streamTracker,
Function<SingleStreamTracker, R> converter) {
StreamTracker streamTracker, Function<SingleStreamTracker, R> converter) {
if (streamTracker instanceof MultiStreamTracker) {
return Either.left((MultiStreamTracker) streamTracker);
} else if (streamTracker instanceof SingleStreamTracker) {
@ -49,5 +48,4 @@ public final class DeprecationUtils {
throw new IllegalArgumentException("Unhandled StreamTracker: " + streamTracker);
}
}
}

20
amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/DiagnosticUtils.java
View file

@ -15,12 +15,12 @@
package software.amazon.kinesis.common;
import org.slf4j.Logger;
import software.amazon.kinesis.annotations.KinesisClientInternalApi;
import java.time.Duration;
import java.time.Instant;
import org.slf4j.Logger;
import software.amazon.kinesis.annotations.KinesisClientInternalApi;
import static software.amazon.kinesis.lifecycle.ShardConsumer.MAX_TIME_BETWEEN_REQUEST_RESPONSE;
@KinesisClientInternalApi
@ -32,18 +32,22 @@ public class DiagnosticUtils {
* @param enqueueTimestamp of the event submitted to the executor service
* @param log Slf4j Logger from RecordPublisher to log the events
*/
public static void takeDelayedDeliveryActionIfRequired(String resourceIdentifier, Instant enqueueTimestamp, Logger log) {
final long durationBetweenEnqueueAndAckInMillis = Duration
.between(enqueueTimestamp, Instant.now()).toMillis();
public static void takeDelayedDeliveryActionIfRequired(
String resourceIdentifier, Instant enqueueTimestamp, Logger log) {
final long durationBetweenEnqueueAndAckInMillis =
Duration.between(enqueueTimestamp, Instant.now()).toMillis();
if (durationBetweenEnqueueAndAckInMillis > 11000) {
// The above condition logs the warn msg if the delivery time exceeds 11 seconds.
log.warn(
"{}: Record delivery time to shard consumer is high at {} millis. Check the ExecutorStateEvent logs"
+ " to see the state of the executor service. Also check if the RecordProcessor's processing "
+ "time is high. ",
resourceIdentifier, durationBetweenEnqueueAndAckInMillis);
resourceIdentifier,
durationBetweenEnqueueAndAckInMillis);
} else if (log.isDebugEnabled()) {
log.debug("{}: Record delivery time to shard consumer is {} millis", resourceIdentifier,
log.debug(
"{}: Record delivery time to shard consumer is {} millis",
resourceIdentifier,
durationBetweenEnqueueAndAckInMillis);
}
}

1
amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/FutureUtils.java
View file

@ -31,5 +31,4 @@ public class FutureUtils {
throw te;
}
}
}

24
amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/HashKeyRangeForLease.java
View file

@ -15,26 +15,30 @@
package software.amazon.kinesis.common;
import java.math.BigInteger;
import lombok.NonNull;
import lombok.Value;
import lombok.experimental.Accessors;
import org.apache.commons.lang3.Validate;
import software.amazon.awssdk.services.kinesis.model.HashKeyRange;
import java.math.BigInteger;
@Value @Accessors(fluent = true)
/**
* Lease POJO to hold the starting hashkey range and ending hashkey range of kinesis shards.
*/
@Accessors(fluent = true)
@Value
public class HashKeyRangeForLease {
private final BigInteger startingHashKey;
private final BigInteger endingHashKey;
public HashKeyRangeForLease(BigInteger startingHashKey, BigInteger endingHashKey) {
Validate.isTrue(startingHashKey.compareTo(endingHashKey) < 0,
"StartingHashKey %s must be less than EndingHashKey %s ", startingHashKey, endingHashKey);
Validate.isTrue(
startingHashKey.compareTo(endingHashKey) < 0,
"StartingHashKey %s must be less than EndingHashKey %s ",
startingHashKey,
endingHashKey);
this.startingHashKey = startingHashKey;
this.endingHashKey = endingHashKey;
}
@ -64,11 +68,15 @@ public class HashKeyRangeForLease {
* @param endingHashKeyStr
* @return HashKeyRangeForLease
*/
public static HashKeyRangeForLease deserialize(@NonNull String startingHashKeyStr, @NonNull String endingHashKeyStr) {
public static HashKeyRangeForLease deserialize(
@NonNull String startingHashKeyStr, @NonNull String endingHashKeyStr) {
final BigInteger startingHashKey = new BigInteger(startingHashKeyStr);
final BigInteger endingHashKey = new BigInteger(endingHashKeyStr);
Validate.isTrue(startingHashKey.compareTo(endingHashKey) < 0,
"StartingHashKey %s must be less than EndingHashKey %s ", startingHashKeyStr, endingHashKeyStr);
Validate.isTrue(
startingHashKey.compareTo(endingHashKey) < 0,
"StartingHashKey %s must be less than EndingHashKey %s ",
startingHashKeyStr,
endingHashKeyStr);
return new HashKeyRangeForLease(startingHashKey, endingHashKey);
}

7
amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/InitialPositionInStreamExtended.java
View file

@ -14,16 +14,17 @@
*/
package software.amazon.kinesis.common;
import java.util.Date;
import lombok.EqualsAndHashCode;
import lombok.ToString;
import java.util.Date;
/**
* Class that houses the entities needed to specify the position in the stream from where a new application should
* start.
*/
@ToString @EqualsAndHashCode
@ToString
@EqualsAndHashCode
public class InitialPositionInStreamExtended {
private final InitialPositionInStream position;

13
amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/KinesisClientUtil.java
View file

@ -15,14 +15,14 @@
package software.amazon.kinesis.common;
import java.time.Duration;
import software.amazon.awssdk.http.Protocol;
import software.amazon.awssdk.http.nio.netty.Http2Configuration;
import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient;
import software.amazon.awssdk.services.kinesis.KinesisAsyncClient;
import software.amazon.awssdk.services.kinesis.KinesisAsyncClientBuilder;
import java.time.Duration;
/**
* Utility to setup KinesisAsyncClient to be used with KCL.
*/
@ -42,9 +42,12 @@ public class KinesisClientUtil {
}
public static KinesisAsyncClientBuilder adjustKinesisClientBuilder(KinesisAsyncClientBuilder builder) {
return builder.httpClientBuilder(NettyNioAsyncHttpClient.builder().maxConcurrency(Integer.MAX_VALUE)
.http2Configuration(Http2Configuration.builder().initialWindowSize(INITIAL_WINDOW_SIZE_BYTES)
.healthCheckPingPeriod(Duration.ofMillis(HEALTH_CHECK_PING_PERIOD_MILLIS)).build())
return builder.httpClientBuilder(NettyNioAsyncHttpClient.builder()
.maxConcurrency(Integer.MAX_VALUE)
.http2Configuration(Http2Configuration.builder()
.initialWindowSize(INITIAL_WINDOW_SIZE_BYTES)
.healthCheckPingPeriod(Duration.ofMillis(HEALTH_CHECK_PING_PERIOD_MILLIS))
.build())
.protocol(Protocol.HTTP2));
}
}

11
amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/KinesisRequestsBuilder.java
View file

@ -63,12 +63,11 @@ public class KinesisRequestsBuilder {
@SuppressWarnings("unchecked")
private static <T extends AwsRequest.Builder> T appendUserAgent(final T builder) {
return (T) builder
.overrideConfiguration(
AwsRequestOverrideConfiguration.builder()
.addApiName(ApiName.builder().name(RetrievalConfig.KINESIS_CLIENT_LIB_USER_AGENT)
.version(RetrievalConfig.KINESIS_CLIENT_LIB_USER_AGENT_VERSION).build())
return (T) builder.overrideConfiguration(AwsRequestOverrideConfiguration.builder()
.addApiName(ApiName.builder()
.name(RetrievalConfig.KINESIS_CLIENT_LIB_USER_AGENT)
.version(RetrievalConfig.KINESIS_CLIENT_LIB_USER_AGENT_VERSION)
.build())
.build());
}
}

2
amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/LeaseCleanupConfig.java
View file

@ -24,7 +24,7 @@ import lombok.experimental.Accessors;
*/
@Builder
@Getter
@Accessors(fluent=true)
@Accessors(fluent = true)
public class LeaseCleanupConfig {
/**
* Interval at which to run lease cleanup thread.

8
amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/RequestDetails.java
View file

@ -15,11 +15,11 @@
package software.amazon.kinesis.common;
import lombok.experimental.Accessors;
import java.util.Optional;
@Accessors(fluent=true)
import lombok.experimental.Accessors;
@Accessors(fluent = true)
public class RequestDetails {
/**
@ -62,6 +62,4 @@ public class RequestDetails {
public String toString() {
return String.format("request id - %s, timestamp - %s", getRequestId(), getTimestamp());
}
}

9
amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/StreamConfig.java
View file

@ -15,15 +15,20 @@
package software.amazon.kinesis.common;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.experimental.Accessors;
@AllArgsConstructor
@RequiredArgsConstructor
@Data
@Accessors(fluent = true)
public class StreamConfig {
@NonNull
private final StreamIdentifier streamIdentifier;
private final InitialPositionInStreamExtended initialPositionInStreamExtended;
private String consumerArn;
}

177
amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/StreamIdentifier.java
View file

@ -15,45 +15,74 @@
package software.amazon.kinesis.common;
import com.google.common.base.Joiner;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import lombok.AccessLevel;
import lombok.Builder;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.NonNull;
import lombok.experimental.Accessors;
import software.amazon.awssdk.arns.Arn;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.utils.Validate;
import java.util.Optional;
import java.util.regex.Pattern;
@EqualsAndHashCode @Getter @Accessors(fluent = true)
@Builder(access = AccessLevel.PRIVATE)
@EqualsAndHashCode
@Getter
@Accessors(fluent = true)
public class StreamIdentifier {
private final Optional<String> accountIdOptional;
@Builder.Default
private final Optional<String> accountIdOptional = Optional.empty();
@NonNull
private final String streamName;
private final Optional<Long> streamCreationEpochOptional;
private static final String DELIMITER = ":";
private static final Pattern PATTERN = Pattern.compile(".*" + ":" + ".*" + ":" + "[0-9]*");
@Builder.Default
private final Optional<Long> streamCreationEpochOptional = Optional.empty();
private StreamIdentifier(@NonNull String accountId, @NonNull String streamName, @NonNull Long streamCreationEpoch) {
this.accountIdOptional = Optional.of(accountId);
this.streamName = streamName;
this.streamCreationEpochOptional = Optional.of(streamCreationEpoch);
}
@Builder.Default
@EqualsAndHashCode.Exclude
private final Optional<Arn> streamArnOptional = Optional.empty();
private StreamIdentifier(@NonNull String streamName) {
this.accountIdOptional = Optional.empty();
this.streamName = streamName;
this.streamCreationEpochOptional = Optional.empty();
}
/**
* Pattern for a serialized {@link StreamIdentifier}. The valid format is
* {@code <accountId>:<streamName>:<creationEpoch>}.
*/
private static final Pattern STREAM_IDENTIFIER_PATTERN =
Pattern.compile("(?<accountId>[0-9]+):(?<streamName>[^:]+):(?<creationEpoch>[0-9]+)");
/**
* Pattern for a stream ARN. The valid format is
* {@code arn:aws:kinesis:<region>:<accountId>:stream:<streamName>}
* where {@code region} is the id representation of a {@link Region}.
*/
private static final Pattern STREAM_ARN_PATTERN = Pattern.compile(
"arn:aws[^:]*:kinesis:(?<region>[-a-z0-9]+):(?<accountId>[0-9]{12}):stream/(?<streamName>.+)");
/**
* Serialize the current StreamIdentifier instance.
* @return
*
* @return a String of {@code account:stream:creationEpoch} in multi-stream mode
* or {@link #streamName} in single-stream mode.
*/
public String serialize() {
return accountIdOptional.isPresent() ?
Joiner.on(DELIMITER).join(accountIdOptional.get(), streamName, streamCreationEpochOptional.get()) :
streamName;
if (!streamCreationEpochOptional.isPresent()) {
// creation epoch is expected to be empty in single-stream mode
return streamName;
}
final char delimiter = ':';
final StringBuilder sb = new StringBuilder()
.append(accountIdOptional.get())
.append(delimiter)
.append(streamName)
.append(delimiter)
.append(streamCreationEpochOptional.get());
return sb.toString();
}
@Override
@ -62,27 +91,105 @@ public class StreamIdentifier {
}
/**
* Create a multi stream instance for StreamIdentifier from serialized stream identifier.
* The serialized stream identifier should be of the format account:stream:creationepoch
* @param streamIdentifierSer
* @return StreamIdentifier
* Create a multi stream instance for StreamIdentifier from serialized stream identifier
* of format {@link #STREAM_IDENTIFIER_PATTERN}
*
* @param streamIdentifierSer a String of {@code account:stream:creationEpoch}
* @return StreamIdentifier with {@link #accountIdOptional} and {@link #streamCreationEpochOptional} present
*/
public static StreamIdentifier multiStreamInstance(String streamIdentifierSer) {
if (PATTERN.matcher(streamIdentifierSer).matches()) {
final String[] split = streamIdentifierSer.split(DELIMITER);
return new StreamIdentifier(split[0], split[1], Long.parseLong(split[2]));
} else {
throw new IllegalArgumentException("Unable to deserialize StreamIdentifier from " + streamIdentifierSer);
final Matcher matcher = STREAM_IDENTIFIER_PATTERN.matcher(streamIdentifierSer);
if (matcher.matches()) {
final String accountId = matcher.group("accountId");
final String streamName = matcher.group("streamName");
final Long creationEpoch = Long.valueOf(matcher.group("creationEpoch"));
validateCreationEpoch(creationEpoch);
return StreamIdentifier.builder()
.accountIdOptional(Optional.of(accountId))
.streamName(streamName)
.streamCreationEpochOptional(Optional.of(creationEpoch))
.build();
}
throw new IllegalArgumentException("Unable to deserialize StreamIdentifier from " + streamIdentifierSer);
}
/**
* Create a multi stream instance for StreamIdentifier from stream {@link Arn}.
*
* @param streamArn an {@link Arn} of format {@link #STREAM_ARN_PATTERN}
* @param creationEpoch Creation epoch of the stream. This value will
* reflect in the lease key and is assumed to be correct. (KCL could
* verify, but that creates issues for both bootstrapping and, with large
* KCL applications, API throttling against DescribeStreamSummary.)
* If this epoch is reused for two identically-named streams in the same
* account -- such as deleting and recreating a stream -- then KCL will
* <b>be unable to differentiate leases between the old and new stream</b>
* since the lease keys collide on this creation epoch.
* @return StreamIdentifier with {@link #accountIdOptional}, {@link #streamCreationEpochOptional},
* and {@link #streamArnOptional} present
*/
public static StreamIdentifier multiStreamInstance(Arn streamArn, long creationEpoch) {
validateArn(streamArn);
validateCreationEpoch(creationEpoch);
return StreamIdentifier.builder()
.accountIdOptional(streamArn.accountId())
.streamName(streamArn.resource().resource())
.streamCreationEpochOptional(Optional.of(creationEpoch))
.streamArnOptional(Optional.of(streamArn))
.build();
}
/**
* Create a single stream instance for StreamIdentifier from stream name.
* @param streamName
* @return StreamIdentifier
*
* @param streamName stream name of a Kinesis stream
*/
public static StreamIdentifier singleStreamInstance(String streamName) {
Validate.notEmpty(streamName, "StreamName should not be empty");
return new StreamIdentifier(streamName);
return StreamIdentifier.builder().streamName(streamName).build();
}
/**
* Create a single stream instance for StreamIdentifier from AWS Kinesis stream {@link Arn}.
*
* @param streamArn AWS ARN of a Kinesis stream
* @return StreamIdentifier with {@link #accountIdOptional} and {@link #streamArnOptional} present
*/
public static StreamIdentifier singleStreamInstance(Arn streamArn) {
validateArn(streamArn);
return StreamIdentifier.builder()
.accountIdOptional(streamArn.accountId())
.streamName(streamArn.resource().resource())
.streamArnOptional(Optional.of(streamArn))
.build();
}
/**
* Verify the streamArn follows the appropriate formatting.
* Throw an exception if it does not.
* @param streamArn
*/
public static void validateArn(Arn streamArn) {
if (!STREAM_ARN_PATTERN.matcher(streamArn.toString()).matches()
|| !streamArn.region().isPresent()) {
throw new IllegalArgumentException("Invalid streamArn " + streamArn);
}
}
/**
* Verify creationEpoch is greater than 0.
* Throw an exception if it is not.
* @param creationEpoch
*/
private static void validateCreationEpoch(long creationEpoch) {
if (creationEpoch <= 0) {
throw new IllegalArgumentException("Creation epoch must be > 0; received " + creationEpoch);
}
}
}

1
amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/CoordinatorConfig.java
View file

@ -96,5 +96,4 @@ public class CoordinatorConfig {
* <p>Default value: 1000L</p>
*/
private long schedulerInitializationBackoffTimeMillis = 1000L;
}

37
amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/DeletedStreamListProvider.java Normal file
View file

@ -0,0 +1,37 @@
package software.amazon.kinesis.coordinator;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import lombok.extern.slf4j.Slf4j;
import software.amazon.kinesis.common.StreamIdentifier;
/**
* This class is used for storing in-memory set of streams which are no longer existing (deleted) and needs to be
* cleaned up from KCL's in memory state.
*/
@Slf4j
public class DeletedStreamListProvider {
private final Set<StreamIdentifier> deletedStreams;
public DeletedStreamListProvider() {
deletedStreams = ConcurrentHashMap.newKeySet();
}
public void add(StreamIdentifier streamIdentifier) {
log.info("Added {}", streamIdentifier);
deletedStreams.add(streamIdentifier);
}
/**
* Method returns and empties the current set of streams
* @return set of deleted Streams
*/
public Set<StreamIdentifier> purgeAllDeletedStream() {
final Set<StreamIdentifier> response = new HashSet<>(deletedStreams);
deletedStreams.removeAll(response);
return response;
}
}

43
amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/DeterministicShuffleShardSyncLeaderDecider.java
View file

@ -26,6 +26,7 @@ import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.BooleanSupplier;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import software.amazon.awssdk.utils.CollectionUtils;
import software.amazon.kinesis.leases.Lease;
@ -45,8 +46,7 @@ import software.amazon.kinesis.leases.exceptions.ProvisionedThroughputException;
* This ensures redundancy for shard-sync during host failures.
*/
@Slf4j
class DeterministicShuffleShardSyncLeaderDecider
implements LeaderDecider {
class DeterministicShuffleShardSyncLeaderDecider implements LeaderDecider {
// Fixed seed so that the shuffle order is preserved across workers
static final int DETERMINISTIC_SHUFFLE_SEED = 1947;
@ -67,13 +67,11 @@ class DeterministicShuffleShardSyncLeaderDecider
* @param leaderElectionThreadPool Thread-pool to be used for leaderElection.
* @param numPeriodicShardSyncWorkers Number of leaders that will be elected to perform periodic shard syncs.
*/
DeterministicShuffleShardSyncLeaderDecider(LeaseRefresher leaseRefresher,
ScheduledExecutorService leaderElectionThreadPool,
int numPeriodicShardSyncWorkers) {
this(leaseRefresher,
leaderElectionThreadPool,
numPeriodicShardSyncWorkers,
new ReentrantReadWriteLock());
DeterministicShuffleShardSyncLeaderDecider(
LeaseRefresher leaseRefresher,
ScheduledExecutorService leaderElectionThreadPool,
int numPeriodicShardSyncWorkers) {
this(leaseRefresher, leaderElectionThreadPool, numPeriodicShardSyncWorkers, new ReentrantReadWriteLock());
}
/**
@ -82,10 +80,11 @@ class DeterministicShuffleShardSyncLeaderDecider
* @param numPeriodicShardSyncWorkers Number of leaders that will be elected to perform periodic shard syncs.
* @param readWriteLock Mechanism to lock for reading and writing of critical components
*/
DeterministicShuffleShardSyncLeaderDecider(LeaseRefresher leaseRefresher,
ScheduledExecutorService leaderElectionThreadPool,
int numPeriodicShardSyncWorkers,
ReadWriteLock readWriteLock) {
DeterministicShuffleShardSyncLeaderDecider(
LeaseRefresher leaseRefresher,
ScheduledExecutorService leaderElectionThreadPool,
int numPeriodicShardSyncWorkers,
ReadWriteLock readWriteLock) {
this.leaseRefresher = leaseRefresher;
this.leaderElectionThreadPool = leaderElectionThreadPool;
this.numPeriodicShardSyncWorkers = numPeriodicShardSyncWorkers;
@ -101,8 +100,12 @@ class DeterministicShuffleShardSyncLeaderDecider
try {
log.debug("Started leader election at: " + Instant.now());
List<Lease> leases = leaseRefresher.listLeases();
List<String> uniqueHosts = leases.stream().map(Lease::leaseOwner)
.filter(owner -> owner != null).distinct().sorted().collect(Collectors.toList());
List<String> uniqueHosts = leases.stream()
.map(Lease::leaseOwner)
.filter(owner -> owner != null)
.distinct()
.sorted()
.collect(Collectors.toList());
Collections.shuffle(uniqueHosts, new Random(DETERMINISTIC_SHUFFLE_SEED));
int numShardSyncWorkers = Math.min(uniqueHosts.size(), numPeriodicShardSyncWorkers);
@ -137,8 +140,11 @@ class DeterministicShuffleShardSyncLeaderDecider
// The first run will be after a minute.
// We don't need jitter since it is scheduled with a fixed delay and time taken to scan leases
// will be different at different times and on different hosts/workers.
leaderElectionThreadPool.scheduleWithFixedDelay(this::electLeaders, ELECTION_INITIAL_DELAY_MILLIS,
ELECTION_SCHEDULING_INTERVAL_MILLIS, TimeUnit.MILLISECONDS);
leaderElectionThreadPool.scheduleWithFixedDelay(
this::electLeaders,
ELECTION_INITIAL_DELAY_MILLIS,
ELECTION_SCHEDULING_INTERVAL_MILLIS,
TimeUnit.MILLISECONDS);
}
return executeConditionCheckWithReadLock(() -> isWorkerLeaderForShardSync(workerId));
@ -152,7 +158,8 @@ class DeterministicShuffleShardSyncLeaderDecider
log.info("Successfully stopped leader election on the worker");
} else {
leaderElectionThreadPool.shutdownNow();
log.info(String.format("Stopped leader election thread after awaiting termination for %d milliseconds",
log.info(String.format(
"Stopped leader election thread after awaiting termination for %d milliseconds",
AWAIT_TERMINATION_MILLIS));
}

4
amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/DiagnosticEventFactory.java
View file

@ -15,11 +15,11 @@
package software.amazon.kinesis.coordinator;
import java.util.concurrent.ExecutorService;
import lombok.NoArgsConstructor;
import software.amazon.kinesis.leases.LeaseCoordinator;
import java.util.concurrent.ExecutorService;
/**
* Creates {@link DiagnosticEvent}s for logging and visibility
*/

6
amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/ExecutorStateEvent.java
View file

@ -15,15 +15,15 @@
package software.amazon.kinesis.coordinator;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadPoolExecutor;
import lombok.Getter;
import lombok.ToString;
import lombok.extern.slf4j.Slf4j;
import software.amazon.kinesis.annotations.KinesisClientInternalApi;
import software.amazon.kinesis.leases.LeaseCoordinator;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadPoolExecutor;
@Getter
@ToString(exclude = "isThreadPoolExecutor")
@Slf4j

12
amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/GracefulShutdownContext.java
View file

@ -14,22 +14,22 @@
*/
package software.amazon.kinesis.coordinator;
import java.util.concurrent.CountDownLatch;
import lombok.Builder;
import lombok.Data;
import lombok.experimental.Accessors;
import java.util.concurrent.CountDownLatch;
@Data
@Builder
@Accessors(fluent = true)
class GracefulShutdownContext {
private final CountDownLatch shutdownCompleteLatch;
private final CountDownLatch notificationCompleteLatch;
private final CountDownLatch finalShutdownLatch;
private final Scheduler scheduler;
static GracefulShutdownContext SHUTDOWN_ALREADY_COMPLETED = new GracefulShutdownContext(null, null, null);
boolean isShutdownAlreadyCompleted() {
boolean isRecordProcessorShutdownComplete() {
return shutdownCompleteLatch == null && notificationCompleteLatch == null && scheduler == null;
}
}

Some files were not shown because too many files have changed in this diff Show more