e2a945d824
The processing Kinesis gets stuck after splitting shard. The
reason is that the app doesn't do mandatory checkpoint.
KCL document states:
// When the value of {@link ShutdownInput#getShutdownReason()} is
// {@link com.amazonaws.services.kinesis.clientlibrary.lib.worker.ShutdownReason#TERMINATE} it is required that you
// checkpoint. Failure to do so will result in an IllegalArgumentException, and the KCL no longer making progress.
Also, fix shard lease to prevent one host takes more shard than
its configuration allowed.
Jira CNA-1701
Change-Id: Icbdacaf347c7a67b5793647ad05ff93cca629741
63 lines
1.9 KiB
Go
63 lines
1.9 KiB
Go
package worker
|
|
|
|
import (
|
|
"github.com/aws/aws-sdk-go/aws"
|
|
|
|
kcl "vmware.com/cascade-kinesis-client/clientlibrary/interfaces"
|
|
)
|
|
|
|
type (
|
|
|
|
/* Objects of this class are prepared to checkpoint at a specific sequence number. They use an
|
|
* IRecordProcessorCheckpointer to do the actual checkpointing, so their checkpoint is subject to the same 'didn't go
|
|
* backwards' validation as a normal checkpoint.
|
|
*/
|
|
PreparedCheckpointer struct {
|
|
pendingCheckpointSequenceNumber *kcl.ExtendedSequenceNumber
|
|
checkpointer kcl.IRecordProcessorCheckpointer
|
|
}
|
|
|
|
/**
|
|
* This class is used to enable RecordProcessors to checkpoint their progress.
|
|
* The Amazon Kinesis Client Library will instantiate an object and provide a reference to the application
|
|
* RecordProcessor instance. Amazon Kinesis Client Library will create one instance per shard assignment.
|
|
*/
|
|
RecordProcessorCheckpointer struct {
|
|
shard *shardStatus
|
|
checkpoint Checkpointer
|
|
}
|
|
)
|
|
|
|
func NewRecordProcessorCheckpoint(shard *shardStatus, checkpoint Checkpointer) kcl.IRecordProcessorCheckpointer {
|
|
return &RecordProcessorCheckpointer{
|
|
shard: shard,
|
|
checkpoint: checkpoint,
|
|
}
|
|
}
|
|
|
|
func (pc *PreparedCheckpointer) GetPendingCheckpoint() *kcl.ExtendedSequenceNumber {
|
|
return pc.pendingCheckpointSequenceNumber
|
|
}
|
|
|
|
func (pc *PreparedCheckpointer) Checkpoint() error {
|
|
return pc.checkpointer.Checkpoint(pc.pendingCheckpointSequenceNumber.SequenceNumber)
|
|
}
|
|
|
|
func (rc *RecordProcessorCheckpointer) Checkpoint(sequenceNumber *string) error {
|
|
rc.shard.mux.Lock()
|
|
|
|
// checkpoint the last sequence of a closed shard
|
|
if sequenceNumber == nil {
|
|
rc.shard.Checkpoint = SHARD_END
|
|
} else {
|
|
rc.shard.Checkpoint = aws.StringValue(sequenceNumber)
|
|
}
|
|
|
|
rc.shard.mux.Unlock()
|
|
return rc.checkpoint.CheckpointSequence(rc.shard)
|
|
}
|
|
|
|
func (rc *RecordProcessorCheckpointer) PrepareCheckpoint(sequenceNumber *string) (kcl.IPreparedCheckpointer, error) {
|
|
return &PreparedCheckpointer{}, nil
|
|
|
|
}
|