Ignore IDE files and fix code based Gometalinter (#63)
- scanError.Error is removed since it is not used. - session.New() is deprecated, The NewKinesisClient's function signature change so it can returns the error from the NewSession().
This commit is contained in:
parent
cf5d22abff
commit
a1239221d8
4 changed files with 21 additions and 12 deletions
3
.gitignore
vendored
3
.gitignore
vendored
|
|
@ -46,3 +46,6 @@ prof.mem
|
||||||
# VSCode files
|
# VSCode files
|
||||||
/.vscode
|
/.vscode
|
||||||
/**/debug
|
/**/debug
|
||||||
|
|
||||||
|
# Goland files
|
||||||
|
.idea/
|
||||||
15
client.go
15
client.go
|
|
@ -20,7 +20,7 @@ func WithKinesis(svc kinesisiface.KinesisAPI) ClientOption {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// WithStartFrmLatest will make sure the client start consuming
|
// WithStartFromLatest will make sure the client start consuming
|
||||||
// events starting from the most recent event in kinesis. This
|
// events starting from the most recent event in kinesis. This
|
||||||
// option discards the checkpoints.
|
// option discards the checkpoints.
|
||||||
func WithStartFromLatest() ClientOption {
|
func WithStartFromLatest() ClientOption {
|
||||||
|
|
@ -30,18 +30,21 @@ func WithStartFromLatest() ClientOption {
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewKinesisClient returns client to interface with Kinesis stream
|
// NewKinesisClient returns client to interface with Kinesis stream
|
||||||
func NewKinesisClient(opts ...ClientOption) *KinesisClient {
|
func NewKinesisClient(opts ...ClientOption) (*KinesisClient, error) {
|
||||||
kc := &KinesisClient{}
|
kc := &KinesisClient{}
|
||||||
|
|
||||||
for _, opt := range opts {
|
for _, opt := range opts {
|
||||||
opt(kc)
|
opt(kc)
|
||||||
}
|
}
|
||||||
|
newSession, err := session.NewSession(aws.NewConfig())
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
if kc.svc == nil {
|
if kc.svc == nil {
|
||||||
kc.svc = kinesis.New(session.New(aws.NewConfig()))
|
kc.svc = kinesis.New(newSession)
|
||||||
}
|
}
|
||||||
|
|
||||||
return kc
|
return kc, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// KinesisClient acts as wrapper around Kinesis client
|
// KinesisClient acts as wrapper around Kinesis client
|
||||||
|
|
@ -61,7 +64,7 @@ func (c *KinesisClient) GetShardIDs(streamName string) ([]string, error) {
|
||||||
return nil, fmt.Errorf("describe stream error: %v", err)
|
return nil, fmt.Errorf("describe stream error: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
ss := []string{}
|
var ss []string
|
||||||
for _, shard := range resp.StreamDescription.Shards {
|
for _, shard := range resp.StreamDescription.Shards {
|
||||||
ss = append(ss, *shard.ShardId)
|
ss = append(ss, *shard.ShardId)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -87,13 +87,18 @@ func New(streamName string, opts ...Option) (*Consumer, error) {
|
||||||
return nil, fmt.Errorf("must provide stream name")
|
return nil, fmt.Errorf("must provide stream name")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
kc, err := NewKinesisClient()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
// new consumer with no-op checkpoint, counter, and logger
|
// new consumer with no-op checkpoint, counter, and logger
|
||||||
c := &Consumer{
|
c := &Consumer{
|
||||||
streamName: streamName,
|
streamName: streamName,
|
||||||
checkpoint: &noopCheckpoint{},
|
checkpoint: &noopCheckpoint{},
|
||||||
counter: &noopCounter{},
|
counter: &noopCounter{},
|
||||||
logger: NewDefaultLogger(),
|
logger: NewDefaultLogger(),
|
||||||
client: NewKinesisClient(),
|
client: kc,
|
||||||
}
|
}
|
||||||
|
|
||||||
// override defaults
|
// override defaults
|
||||||
|
|
@ -178,8 +183,6 @@ func (c *Consumer) ScanShard(ctx context.Context, shardID string, fn func(*Recor
|
||||||
// loop records
|
// loop records
|
||||||
for r := range recc {
|
for r := range recc {
|
||||||
scanError := fn(r)
|
scanError := fn(r)
|
||||||
// It will be nicer if this can be reported with checkpoint error
|
|
||||||
err = scanError.Error
|
|
||||||
|
|
||||||
// Skip invalid state
|
// Skip invalid state
|
||||||
if scanError.StopScan && scanError.SkipCheckpoint {
|
if scanError.StopScan && scanError.SkipCheckpoint {
|
||||||
|
|
|
||||||
|
|
@ -3,11 +3,11 @@ package consumer
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io/ioutil"
|
|
||||||
"log"
|
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"errors"
|
||||||
|
|
||||||
"github.com/aws/aws-sdk-go/aws"
|
"github.com/aws/aws-sdk-go/aws"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -39,7 +39,7 @@ func TestScanShard(t *testing.T) {
|
||||||
client: client,
|
client: client,
|
||||||
checkpoint: ckp,
|
checkpoint: ckp,
|
||||||
counter: ctr,
|
counter: ctr,
|
||||||
logger: log.New(ioutil.Discard, "", log.LstdFlags),
|
logger: NewDefaultLogger(),
|
||||||
}
|
}
|
||||||
|
|
||||||
// callback fn simply appends the record data to result string
|
// callback fn simply appends the record data to result string
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue