99 lines
2.2 KiB
Go
99 lines
2.2 KiB
Go
|
package connector
|
||
|
|
|
||
|
|
import (
|
||
|
|
"math"
|
||
|
|
"net"
|
||
|
|
"net/url"
|
||
|
|
"reflect"
|
||
|
|
"regexp"
|
||
|
|
"time"
|
||
|
|
|
||
|
|
"github.com/lib/pq"
|
||
|
|
"github.com/sendgridlabs/go-kinesis"
|
||
|
|
)
|
||
|
|
|
||
|
|
type isRecoverableErrorFunc func(error) bool
|
||
|
|
|
||
|
|
func kinesisIsRecoverableError(err error) bool {
|
||
|
|
recoverableErrorCodes := map[string]bool{
|
||
|
|
"ProvisionedThroughputExceededException": true,
|
||
|
|
"InternalFailure": true,
|
||
|
|
"Throttling": true,
|
||
|
|
"ServiceUnavailable": true,
|
||
|
|
}
|
||
|
|
r := false
|
||
|
|
cErr, ok := err.(*kinesis.Error)
|
||
|
|
if ok && recoverableErrorCodes[cErr.Code] == true {
|
||
|
|
r = true
|
||
|
|
}
|
||
|
|
return r
|
||
|
|
}
|
||
|
|
|
||
|
|
func urlIsRecoverableError(err error) bool {
|
||
|
|
r := false
|
||
|
|
_, ok := err.(*url.Error)
|
||
|
|
if ok {
|
||
|
|
r = true
|
||
|
|
}
|
||
|
|
return r
|
||
|
|
}
|
||
|
|
|
||
|
|
func netIsRecoverableError(err error) bool {
|
||
|
|
recoverableErrors := map[string]bool{
|
||
|
|
"connection reset by peer": true,
|
||
|
|
}
|
||
|
|
cErr, ok := err.(*net.OpError)
|
||
|
|
if ok && recoverableErrors[cErr.Err.Error()] == true {
|
||
|
|
return true
|
||
|
|
}
|
||
|
|
return false
|
||
|
|
}
|
||
|
|
|
||
|
|
var redshiftRecoverableErrors = []*regexp.Regexp{
|
||
|
|
regexp.MustCompile("The specified S3 prefix '.*?' does not exist"),
|
||
|
|
}
|
||
|
|
|
||
|
|
func redshiftIsRecoverableError(err error) bool {
|
||
|
|
r := false
|
||
|
|
if cErr, ok := err.(pq.Error); ok {
|
||
|
|
for _, re := range redshiftRecoverableErrors {
|
||
|
|
if re.MatchString(cErr.Message) {
|
||
|
|
r = true
|
||
|
|
break
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return r
|
||
|
|
}
|
||
|
|
|
||
|
|
var isRecoverableErrors = []isRecoverableErrorFunc{
|
||
|
|
kinesisIsRecoverableError, netIsRecoverableError, urlIsRecoverableError, redshiftIsRecoverableError,
|
||
|
|
}
|
||
|
|
|
||
|
|
// this determines whether the error is recoverable
|
||
|
|
func isRecoverableError(err error) bool {
|
||
|
|
r := false
|
||
|
|
|
||
|
|
logger.Printf("isRecoverableError, type %s, value (%#v)\n", reflect.TypeOf(err).String(), err)
|
||
|
|
|
||
|
|
for _, errF := range isRecoverableErrors {
|
||
|
|
r = errF(err)
|
||
|
|
if r {
|
||
|
|
break
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
return r
|
||
|
|
}
|
||
|
|
|
||
|
|
// handle the aws exponential backoff
|
||
|
|
// wait up to 5 minutes based on the aws exponential backoff algorithm
|
||
|
|
// http://docs.aws.amazon.com/general/latest/gr/api-retries.html
|
||
|
|
func handleAwsWaitTimeExp(attempts int) {
|
||
|
|
if attempts > 0 {
|
||
|
|
waitTime := time.Duration(math.Min(100*math.Pow(2, float64(attempts)), 300000)) * time.Millisecond
|
||
|
|
logger.Printf("handleAwsWaitTimeExp: %s\n", waitTime.String())
|
||
|
|
time.Sleep(waitTime)
|
||
|
|
}
|
||
|
|
}
|