Fix retry logic for dynamodb (#83)

Adding min/max retry and throttle delay for the retryer.
Also, increase the max retries to 10 which is inline with
dynamodb default retry count.

Signed-off-by: Tao Jiang <taoj@vmware.com>

clientlibrary/checkpoint/dynamodb-checkpointer.go

@@ -48,7 +48,7 @@ const (
 	ErrInvalidDynamoDBSchema = "The DynamoDB schema is invalid and may need to be re-created"
 
 	// NumMaxRetries is the max times of doing retry
-	NumMaxRetries = 5
+	NumMaxRetries = 10
 )
 
 // DynamoCheckpoint implements the Checkpoint interface using DynamoDB as a backend
@@ -92,7 +92,13 @@ func (checkpointer *DynamoCheckpoint) Init() error {
 		Region:      aws.String(checkpointer.kclConfig.RegionName),
 		Endpoint:    aws.String(checkpointer.kclConfig.DynamoDBEndpoint),
 		Credentials: checkpointer.kclConfig.DynamoDBCredentials,
-		Retryer:     client.DefaultRetryer{NumMaxRetries: checkpointer.Retries},
+		Retryer: client.DefaultRetryer{
+			NumMaxRetries:    checkpointer.Retries,
+			MinRetryDelay:    client.DefaultRetryerMinRetryDelay,
+			MinThrottleDelay: client.DefaultRetryerMinThrottleDelay,
+			MaxRetryDelay:    client.DefaultRetryerMaxRetryDelay,
+			MaxThrottleDelay: client.DefaultRetryerMaxRetryDelay,
+		},
 	})
 
 	if err != nil {

test/worker_test.go

@@ -51,7 +51,7 @@ const metricsSystem = "cloudwatch"
 var shardID string
 
 func TestWorker(t *testing.T) {
-	// At miminal. use standard logrus logger
+	// At minimal. use standard logrus logger
 	// log := logger.NewLogrusLogger(logrus.StandardLogger())
 	//
 	// In order to have precise control over logging. Use logger with config