luciano/kinesis-consumer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Get a new shard iterator on error (#32)

Browse source 
* Get a new shard iterator on error
* Check for nil instead of empty string
* Get a new shard iterator on error
This commit is contained in:
Pierre Massat 2017-10-15 17:40:30 -07:00 committed by Harlow Ward
parent c56cefb667
commit c04f3d8a94
1 changed files with 54 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

80
consumer.go
View file

@ -54,30 +54,10 @@ func (c *Consumer) handlerLoop(shardID string, handler Handler) {
MaxRecordCount: c.BufferSize, MaxRecordCount: c.BufferSize,
shardID: shardID, shardID: shardID,
} }
params := &kinesis.GetShardIteratorInput{
ShardId: aws.String(shardID),
StreamName: aws.String(c.StreamName),
}
if c.Checkpoint.CheckpointExists(shardID) {
params.ShardIteratorType = aws.String("AFTER_SEQUENCE_NUMBER")
params.StartingSequenceNumber = aws.String(c.Checkpoint.SequenceNumber())
} else {
params.ShardIteratorType = aws.String("TRIM_HORIZON")
}
resp, err := c.svc.GetShardIterator(params)
if err != nil {
c.Logger.WithError(err).Error("GetShardIterator")
os.Exit(1)
}
shardIterator := resp.ShardIterator
ctx := c.Logger.WithFields(log.Fields{ ctx := c.Logger.WithFields(log.Fields{
"shard": shardID, "shard": shardID,
}) })
shardIterator := c.getShardIterator(shardID)
ctx.Info("processing") ctx.Info("processing")
@ -89,7 +69,9 @@ func (c *Consumer) handlerLoop(shardID string, handler Handler) {
) )
if err != nil { if err != nil {
log.Fatalf("Error GetRecords %v", err) ctx.WithError(err).Error("GetRecords")
shardIterator = c.getShardIterator(shardID)
continue
} }
if len(resp.Records) > 0 { if len(resp.Records) > 0 {
@ -103,11 +85,57 @@ func (c *Consumer) handlerLoop(shardID string, handler Handler) {
buf.Flush() buf.Flush()
} }
} }
} else if resp.NextShardIterator == aws.String("") || shardIterator == resp.NextShardIterator { }
c.Logger.Error("NextShardIterator")
if resp.NextShardIterator == nil || shardIterator == resp.NextShardIterator {
shardIterator = c.getShardIterator(shardID)
} else {
shardIterator = resp.NextShardIterator
}
}
}
func (c *Consumer) getShardIterator(shardID string) *string {
params := &kinesis.GetShardIteratorInput{
ShardId: aws.String(shardID),
StreamName: aws.String(c.StreamName),
}
if c.Checkpoint.CheckpointExists(shardID) {
params.ShardIteratorType = aws.String(string(ShardIteratorAfterSequenceNumber))
params.StartingSequenceNumber = aws.String(c.Checkpoint.SequenceNumber())
} else {
params.ShardIteratorType = aws.String(string(c.ShardIteratorType))
}
resp, err := c.svc.GetShardIterator(params)
if err != nil {
c.Logger.WithError(err).Error("GetShardIterator")
os.Exit(1) os.Exit(1)
} }
shardIterator = resp.NextShardIterator return resp.ShardIterator
} }
func (c *Consumer) getShardIterator(shardID string) *string {
params := &kinesis.GetShardIteratorInput{
ShardId: aws.String(shardID),
StreamName: aws.String(c.StreamName),
}
if c.Checkpoint.CheckpointExists(shardID) {
params.ShardIteratorType = aws.String(string(ShardIteratorAfterSequenceNumber))
params.StartingSequenceNumber = aws.String(c.Checkpoint.SequenceNumber())
} else {
params.ShardIteratorType = aws.String(string(c.ShardIteratorType))
}
resp, err := c.svc.GetShardIterator(params)
if err != nil {
c.Logger.WithError(err).Error("GetShardIterator")
os.Exit(1)
}
return resp.ShardIterator
} }