2f4ff65681
* [#367] Enhanced multi-lang `AWSCredentialsProvider=...` decoder and construction. + added support for external ids (issue #367) + added support for endpoint+region (e.g., STS via VPC) * Multiple multi-lang edits to introduce logging and additional tests. + added `ENDPOINT_REGION` nested key for a simpler Cx experience + deduplicated, and improved, logic w.r.t. CredentialsProvider construction to NOT swallow Exceptions * Relocated `multilang.properties` from `main/resources` to `test/resources`
93 lines
4.5 KiB
Properties
93 lines
4.5 KiB
Properties
# The script that abides by the multi-language protocol. This script will
|
|
# be executed by the MultiLangDaemon, which will communicate with this script
|
|
# over STDIN and STDOUT according to the multi-language protocol.
|
|
executableName = sample_kclpy_app.py
|
|
|
|
# The Stream arn: arn:aws:kinesis:<region>:<account id>:stream/<stream name>
|
|
# Important: streamArn takes precedence over streamName if both are set
|
|
streamArn = arn:aws:kinesis:us-east-5:000000000000:stream/kclpysample
|
|
|
|
# The name of an Amazon Kinesis stream to process.
|
|
# Important: streamArn takes precedence over streamName if both are set
|
|
streamName = kclpysample
|
|
|
|
# Used by the KCL as the name of this application. Will be used as the name
|
|
# of an Amazon DynamoDB table which will store the lease and checkpoint
|
|
# information for workers with this application name
|
|
applicationName = MultiLangTest
|
|
|
|
# Users can change the credentials provider the KCL will use to retrieve credentials.
|
|
# The DefaultAWSCredentialsProviderChain checks several other providers, which is
|
|
# described here:
|
|
# http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html
|
|
AWSCredentialsProvider = DefaultAWSCredentialsProviderChain
|
|
|
|
# Appended to the user agent of the KCL. Does not impact the functionality of the
|
|
# KCL in any other way.
|
|
processingLanguage = python/3.8
|
|
|
|
# Valid options at TRIM_HORIZON or LATEST.
|
|
# See http://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetShardIterator.html#API_GetShardIterator_RequestSyntax
|
|
initialPositionInStream = TRIM_HORIZON
|
|
|
|
# To specify an initial timestamp from which to start processing records, please specify timestamp value for 'initiatPositionInStreamExtended',
|
|
# and uncomment below line with right timestamp value.
|
|
# See more from 'Timestamp' under http://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetShardIterator.html#API_GetShardIterator_RequestSyntax
|
|
#initialPositionInStreamExtended = 1636609142
|
|
|
|
# The following properties are also available for configuring the KCL Worker that is created
|
|
# by the MultiLangDaemon.
|
|
|
|
# The KCL defaults to us-east-1
|
|
regionName = us-east-1
|
|
|
|
# Fail over time in milliseconds. A worker which does not renew it's lease within this time interval
|
|
# will be regarded as having problems and it's shards will be assigned to other workers.
|
|
# For applications that have a large number of shards, this msy be set to a higher number to reduce
|
|
# the number of DynamoDB IOPS required for tracking leases
|
|
failoverTimeMillis = 10000
|
|
|
|
# A worker id that uniquely identifies this worker among all workers using the same applicationName
|
|
# If this isn't provided a MultiLangDaemon instance will assign a unique workerId to itself.
|
|
workerId = "workerId"
|
|
|
|
# Shard sync interval in milliseconds - e.g. wait for this long between shard sync tasks.
|
|
shardSyncIntervalMillis = 60000
|
|
|
|
# Max records to fetch from Kinesis in a single GetRecords call.
|
|
maxRecords = 10000
|
|
|
|
# Idle time between record reads in milliseconds.
|
|
idleTimeBetweenReadsInMillis = 1000
|
|
|
|
# Enables applications flush/checkpoint (if they have some data "in progress", but don't get new data for while)
|
|
callProcessRecordsEvenForEmptyRecordList = false
|
|
|
|
# Interval in milliseconds between polling to check for parent shard completion.
|
|
# Polling frequently will take up more DynamoDB IOPS (when there are leases for shards waiting on
|
|
# completion of parent shards).
|
|
parentShardPollIntervalMillis = 10000
|
|
|
|
# Cleanup leases upon shards completion (don't wait until they expire in Kinesis).
|
|
# Keeping leases takes some tracking/resources (e.g. they need to be renewed, assigned), so by default we try
|
|
# to delete the ones we don't need any longer.
|
|
cleanupLeasesUponShardCompletion = true
|
|
|
|
# Backoff time in milliseconds for Amazon Kinesis Client Library tasks (in the event of failures).
|
|
taskBackoffTimeMillis = 500
|
|
|
|
# Buffer metrics for at most this long before publishing to CloudWatch.
|
|
metricsBufferTimeMillis = 10000
|
|
|
|
# Buffer at most this many metrics before publishing to CloudWatch.
|
|
metricsMaxQueueSize = 10000
|
|
|
|
# KCL will validate client provided sequence numbers with a call to Amazon Kinesis before checkpointing for calls
|
|
# to RecordProcessorCheckpointer#checkpoint(String) by default.
|
|
validateSequenceNumberBeforeCheckpointing = true
|
|
|
|
# The maximum number of active threads for the MultiLangDaemon to permit.
|
|
# If a value is provided then a FixedThreadPool is used with the maximum
|
|
# active threads set to the provided value. If a non-positive integer or no
|
|
# value is provided a CachedThreadPool is used.
|
|
maxActiveThreads = -1
|