Add checkpoint interface for custom checkpoints (#29)
To allow other checkpoint backends we extract a checkpoint interface which future checkpoints can implement. * Add checkpoint interface for custom checkpoints * Create RedisCheckpoint to implement checkpoint interface * Swap out hosie redis library for go-redis Minor changes * Allow configuration of Redis endpoint with env var `REDIS_URL` * Replace gvt with govendor
This commit is contained in:
Harlow Ward
GitHub
parent
cc936aed04
commit
fedb6812fb
10 changed files with 469 additions and 271 deletions
2
.gitignore
vendored
2
.gitignore
vendored
|
|
@ -38,7 +38,7 @@ tags*
|
||||||
|
|
||||||
# Vendored files
|
# Vendored files
|
||||||
vendor/**
|
vendor/**
|
||||||
!vendor/manifest
|
!vendor/vendor.json
|
||||||
|
|
||||||
# Benchmark files
|
# Benchmark files
|
||||||
prof.cpu
|
prof.cpu
|
||||||
|
|
|
||||||
15
README.md
15
README.md
|
|
@ -36,6 +36,14 @@ func main() {
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Config
|
||||||
|
|
||||||
|
The default behavior for checkpointing uses Redis on localhost. To set a custom Redis URL use ENV vars:
|
||||||
|
|
||||||
|
```
|
||||||
|
REDIS_URL=my-custom-redis-server.com:6379
|
||||||
|
```
|
||||||
|
|
||||||
### Logging
|
### Logging
|
||||||
|
|
||||||
[Apex Log](https://medium.com/@tjholowaychuk/apex-log-e8d9627f4a9a#.5x1uo1767) is used for logging Info. Override the logs format with other [Log Handlers](https://github.com/apex/log/tree/master/_examples). For example using the "json" log handler:
|
[Apex Log](https://medium.com/@tjholowaychuk/apex-log-e8d9627f4a9a#.5x1uo1767) is used for logging Info. Override the logs format with other [Log Handlers](https://github.com/apex/log/tree/master/_examples). For example using the "json" log handler:
|
||||||
|
|
@ -70,14 +78,14 @@ Get the package source:
|
||||||
|
|
||||||
### Fetching Dependencies
|
### Fetching Dependencies
|
||||||
|
|
||||||
Install `gvt`:
|
Install `govendor`:
|
||||||
|
|
||||||
$ export GO15VENDOREXPERIMENT=1
|
$ export GO15VENDOREXPERIMENT=1
|
||||||
$ go get github.com/FiloSottile/gvt
|
$ go get -u github.com/kardianos/govendor
|
||||||
|
|
||||||
Install dependencies into `./vendor/`:
|
Install dependencies into `./vendor/`:
|
||||||
|
|
||||||
$ gvt restore
|
$ govendor sync
|
||||||
|
|
||||||
### Examples
|
### Examples
|
||||||
|
|
||||||
|
|
@ -86,7 +94,6 @@ Use the [seed stream](https://github.com/harlow/kinesis-connectors/tree/master/e
|
||||||
* [Firehose](https://github.com/harlow/kinesis-connectors/tree/master/examples/firehose)
|
* [Firehose](https://github.com/harlow/kinesis-connectors/tree/master/examples/firehose)
|
||||||
* [S3](https://github.com/harlow/kinesis-connectors/tree/master/examples/s3)
|
* [S3](https://github.com/harlow/kinesis-connectors/tree/master/examples/s3)
|
||||||
|
|
||||||
|
|
||||||
## Contributing
|
## Contributing
|
||||||
|
|
||||||
Please see [CONTRIBUTING.md] for more information. Thank you, [contributors]!
|
Please see [CONTRIBUTING.md] for more information. Thank you, [contributors]!
|
||||||
|
|
|
||||||
|
|
@ -1,48 +1,9 @@
|
||||||
package connector
|
package connector
|
||||||
|
|
||||||
import (
|
// Checkpoint interface for functions that checkpoints need to
|
||||||
"fmt"
|
// implement in order to track consumer progress.
|
||||||
|
type Checkpoint interface {
|
||||||
"github.com/hoisie/redis"
|
CheckpointExists(shardID string) bool
|
||||||
)
|
SequenceNumber() string
|
||||||
|
SetCheckpoint(shardID string, sequenceNumber string)
|
||||||
// RedisCheckpoint implements the Checkpont interface.
|
|
||||||
// This class is used to enable the Pipeline.ProcessShard to checkpoint their progress.
|
|
||||||
type Checkpoint struct {
|
|
||||||
AppName string
|
|
||||||
StreamName string
|
|
||||||
|
|
||||||
client redis.Client
|
|
||||||
sequenceNumber string
|
|
||||||
}
|
|
||||||
|
|
||||||
// CheckpointExists determines if a checkpoint for a particular Shard exists.
|
|
||||||
// Typically used to determine whether we should start processing the shard with
|
|
||||||
// TRIM_HORIZON or AFTER_SEQUENCE_NUMBER (if checkpoint exists).
|
|
||||||
func (c *Checkpoint) CheckpointExists(shardID string) bool {
|
|
||||||
val, _ := c.client.Get(c.key(shardID))
|
|
||||||
|
|
||||||
if val != nil && string(val) != "" {
|
|
||||||
c.sequenceNumber = string(val)
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// SequenceNumber returns the current checkpoint stored for the specified shard.
|
|
||||||
func (c *Checkpoint) SequenceNumber() string {
|
|
||||||
return c.sequenceNumber
|
|
||||||
}
|
|
||||||
|
|
||||||
// SetCheckpoint stores a checkpoint for a shard (e.g. sequence number of last record processed by application).
|
|
||||||
// Upon failover, record processing is resumed from this point.
|
|
||||||
func (c *Checkpoint) SetCheckpoint(shardID string, sequenceNumber string) {
|
|
||||||
c.client.Set(c.key(shardID), []byte(sequenceNumber))
|
|
||||||
c.sequenceNumber = sequenceNumber
|
|
||||||
}
|
|
||||||
|
|
||||||
// key generates a unique Redis key for storage of Checkpoint.
|
|
||||||
func (c *Checkpoint) key(shardID string) string {
|
|
||||||
return fmt.Sprintf("%v:checkpoint:%v:%v", c.AppName, c.StreamName, shardID)
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,46 +0,0 @@
|
||||||
package connector
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/bmizerany/assert"
|
|
||||||
"github.com/hoisie/redis"
|
|
||||||
)
|
|
||||||
|
|
||||||
func Test_key(t *testing.T) {
|
|
||||||
c := Checkpoint{
|
|
||||||
AppName: "app",
|
|
||||||
StreamName: "stream",
|
|
||||||
}
|
|
||||||
|
|
||||||
k := c.key("shard")
|
|
||||||
assert.Equal(t, k, "app:checkpoint:stream:shard")
|
|
||||||
}
|
|
||||||
|
|
||||||
func Test_CheckpointExists(t *testing.T) {
|
|
||||||
var rc redis.Client
|
|
||||||
rc.Set("app:checkpoint:stream:shard", []byte("testSeqNum"))
|
|
||||||
c := Checkpoint{
|
|
||||||
AppName: "app",
|
|
||||||
StreamName: "stream",
|
|
||||||
}
|
|
||||||
|
|
||||||
r := c.CheckpointExists("shard")
|
|
||||||
assert.Equal(t, r, true)
|
|
||||||
|
|
||||||
rc.Del("app:checkpoint:stream:shard")
|
|
||||||
}
|
|
||||||
|
|
||||||
func Test_SetCheckpoint(t *testing.T) {
|
|
||||||
var rc redis.Client
|
|
||||||
c := Checkpoint{
|
|
||||||
AppName: "app",
|
|
||||||
StreamName: "stream",
|
|
||||||
}
|
|
||||||
|
|
||||||
c.SetCheckpoint("shard", "testSeqNum")
|
|
||||||
r, _ := rc.Get("app:checkpoint:stream:shard")
|
|
||||||
assert.Equal(t, string(r), "testSeqNum")
|
|
||||||
|
|
||||||
rc.Del("app:checkpoint:stream:shard")
|
|
||||||
}
|
|
||||||
37
config.go
37
config.go
|
|
@ -4,15 +4,19 @@ import (
|
||||||
"os"
|
"os"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
redis "gopkg.in/redis.v5"
|
||||||
|
|
||||||
"github.com/apex/log"
|
"github.com/apex/log"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
defaultBufferSize = 500
|
defaultBufferSize = 500
|
||||||
|
defaultRedisAddr = "127.0.0.1:6379"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Config vars for the application
|
||||||
type Config struct {
|
type Config struct {
|
||||||
// AppName is the application name.
|
// AppName is the application name and checkpoint namespace.
|
||||||
AppName string
|
AppName string
|
||||||
|
|
||||||
// StreamName is the Kinesis stream.
|
// StreamName is the Kinesis stream.
|
||||||
|
|
@ -26,6 +30,9 @@ type Config struct {
|
||||||
|
|
||||||
// Logger is the logger used. Defaults to log.Log.
|
// Logger is the logger used. Defaults to log.Log.
|
||||||
Logger log.Interface
|
Logger log.Interface
|
||||||
|
|
||||||
|
// Checkpoint for tracking progress of consumer.
|
||||||
|
Checkpoint Checkpoint
|
||||||
}
|
}
|
||||||
|
|
||||||
// defaults for configuration.
|
// defaults for configuration.
|
||||||
|
|
@ -60,4 +67,32 @@ func (c *Config) setDefaults() {
|
||||||
if c.FlushInterval == 0 {
|
if c.FlushInterval == 0 {
|
||||||
c.FlushInterval = time.Second
|
c.FlushInterval = time.Second
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if c.Checkpoint == nil {
|
||||||
|
client, err := redisClient()
|
||||||
|
if err != nil {
|
||||||
|
c.Logger.WithError(err).Error("Redis connection failed")
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
c.Checkpoint = &RedisCheckpoint{
|
||||||
|
AppName: c.AppName,
|
||||||
|
StreamName: c.StreamName,
|
||||||
|
client: client,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func redisClient() (*redis.Client, error) {
|
||||||
|
redisURL := os.Getenv("REDIS_URL")
|
||||||
|
if redisURL == "" {
|
||||||
|
redisURL = defaultRedisAddr
|
||||||
|
}
|
||||||
|
client := redis.NewClient(&redis.Options{
|
||||||
|
Addr: redisURL,
|
||||||
|
})
|
||||||
|
_, err := client.Ping().Result()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return client, nil
|
||||||
}
|
}
|
||||||
|
|
|
||||||
11
consumer.go
11
consumer.go
|
|
@ -54,19 +54,14 @@ func (c *Consumer) handlerLoop(shardID string, handler Handler) {
|
||||||
MaxRecordCount: c.BufferSize,
|
MaxRecordCount: c.BufferSize,
|
||||||
}
|
}
|
||||||
|
|
||||||
checkpoint := &Checkpoint{
|
|
||||||
AppName: c.AppName,
|
|
||||||
StreamName: c.StreamName,
|
|
||||||
}
|
|
||||||
|
|
||||||
params := &kinesis.GetShardIteratorInput{
|
params := &kinesis.GetShardIteratorInput{
|
||||||
ShardId: aws.String(shardID),
|
ShardId: aws.String(shardID),
|
||||||
StreamName: aws.String(c.StreamName),
|
StreamName: aws.String(c.StreamName),
|
||||||
}
|
}
|
||||||
|
|
||||||
if checkpoint.CheckpointExists(shardID) {
|
if c.Checkpoint.CheckpointExists(shardID) {
|
||||||
params.ShardIteratorType = aws.String("AFTER_SEQUENCE_NUMBER")
|
params.ShardIteratorType = aws.String("AFTER_SEQUENCE_NUMBER")
|
||||||
params.StartingSequenceNumber = aws.String(checkpoint.SequenceNumber())
|
params.StartingSequenceNumber = aws.String(c.Checkpoint.SequenceNumber())
|
||||||
} else {
|
} else {
|
||||||
params.ShardIteratorType = aws.String("TRIM_HORIZON")
|
params.ShardIteratorType = aws.String("TRIM_HORIZON")
|
||||||
}
|
}
|
||||||
|
|
@ -103,7 +98,7 @@ func (c *Consumer) handlerLoop(shardID string, handler Handler) {
|
||||||
if buf.ShouldFlush() {
|
if buf.ShouldFlush() {
|
||||||
handler.HandleRecords(*buf)
|
handler.HandleRecords(*buf)
|
||||||
ctx.WithField("count", buf.RecordCount()).Info("flushed")
|
ctx.WithField("count", buf.RecordCount()).Info("flushed")
|
||||||
checkpoint.SetCheckpoint(shardID, buf.LastSeq())
|
c.Checkpoint.SetCheckpoint(shardID, buf.LastSeq())
|
||||||
buf.Flush()
|
buf.Flush()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
53
redis_checkpoint.go
Normal file
53
redis_checkpoint.go
Normal file
|
|
@ -0,0 +1,53 @@
|
||||||
|
package connector
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
|
||||||
|
"gopkg.in/redis.v5"
|
||||||
|
)
|
||||||
|
|
||||||
|
// RedisCheckpoint implements the Checkpont interface.
|
||||||
|
// Used to enable the Pipeline.ProcessShard to checkpoint it's progress
|
||||||
|
// while reading records from Kinesis stream.
|
||||||
|
type RedisCheckpoint struct {
|
||||||
|
AppName string
|
||||||
|
StreamName string
|
||||||
|
|
||||||
|
client *redis.Client
|
||||||
|
sequenceNumber string
|
||||||
|
}
|
||||||
|
|
||||||
|
// CheckpointExists determines if a checkpoint for a particular Shard exists.
|
||||||
|
// Typically used to determine whether we should start processing the shard with
|
||||||
|
// TRIM_HORIZON or AFTER_SEQUENCE_NUMBER (if checkpoint exists).
|
||||||
|
func (c *RedisCheckpoint) CheckpointExists(shardID string) bool {
|
||||||
|
val, _ := c.client.Get(c.key(shardID)).Result()
|
||||||
|
|
||||||
|
if val != "" {
|
||||||
|
c.sequenceNumber = val
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// SequenceNumber returns the current checkpoint stored for the specified shard.
|
||||||
|
func (c *RedisCheckpoint) SequenceNumber() string {
|
||||||
|
return c.sequenceNumber
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetCheckpoint stores a checkpoint for a shard (e.g. sequence number of last record processed by application).
|
||||||
|
// Upon failover, record processing is resumed from this point.
|
||||||
|
func (c *RedisCheckpoint) SetCheckpoint(shardID string, sequenceNumber string) {
|
||||||
|
err := c.client.Set(c.key(shardID), sequenceNumber, 0).Err()
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("redis checkpoint set error: %v", err)
|
||||||
|
}
|
||||||
|
c.sequenceNumber = sequenceNumber
|
||||||
|
}
|
||||||
|
|
||||||
|
// key generates a unique Redis key for storage of Checkpoint.
|
||||||
|
func (c *RedisCheckpoint) key(shardID string) string {
|
||||||
|
return fmt.Sprintf("%v:checkpoint:%v:%v", c.AppName, c.StreamName, shardID)
|
||||||
|
}
|
||||||
50
redis_checkpoint_test.go
Normal file
50
redis_checkpoint_test.go
Normal file
|
|
@ -0,0 +1,50 @@
|
||||||
|
package connector
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gopkg.in/redis.v5"
|
||||||
|
)
|
||||||
|
|
||||||
|
var defaultAddr = "127.0.0.1:6379"
|
||||||
|
|
||||||
|
func Test_CheckpointLifecycle(t *testing.T) {
|
||||||
|
client := redis.NewClient(&redis.Options{Addr: defaultAddr})
|
||||||
|
|
||||||
|
c := RedisCheckpoint{
|
||||||
|
AppName: "app",
|
||||||
|
StreamName: "stream",
|
||||||
|
client: client,
|
||||||
|
}
|
||||||
|
|
||||||
|
// set checkpoint
|
||||||
|
c.SetCheckpoint("shard_id", "testSeqNum")
|
||||||
|
|
||||||
|
// checkpoint exists
|
||||||
|
if val := c.CheckpointExists("shard_id"); val != true {
|
||||||
|
t.Fatalf("checkpoint exists expected true, got %t", val)
|
||||||
|
}
|
||||||
|
|
||||||
|
// get checkpoint
|
||||||
|
if val := c.SequenceNumber(); val != "testSeqNum" {
|
||||||
|
t.Fatalf("checkpoint exists expected %s, got %s", "testSeqNum", val)
|
||||||
|
}
|
||||||
|
|
||||||
|
client.Del("app:checkpoint:stream:shard_id")
|
||||||
|
}
|
||||||
|
|
||||||
|
func Test_key(t *testing.T) {
|
||||||
|
client := redis.NewClient(&redis.Options{Addr: defaultAddr})
|
||||||
|
|
||||||
|
c := &RedisCheckpoint{
|
||||||
|
AppName: "app",
|
||||||
|
StreamName: "stream",
|
||||||
|
client: client,
|
||||||
|
}
|
||||||
|
|
||||||
|
expected := "app:checkpoint:stream:shard"
|
||||||
|
|
||||||
|
if val := c.key("shard"); val != expected {
|
||||||
|
t.Fatalf("checkpoint exists expected %s, got %s", expected, val)
|
||||||
|
}
|
||||||
|
}
|
||||||
166
vendor/manifest
vendored
166
vendor/manifest
vendored
|
|
@ -1,166 +0,0 @@
|
||||||
{
|
|
||||||
"version": 0,
|
|
||||||
"dependencies": [
|
|
||||||
{
|
|
||||||
"importpath": "github.com/apex/log",
|
|
||||||
"repository": "https://github.com/apex/log",
|
|
||||||
"vcs": "git",
|
|
||||||
"revision": "c20bfcdb771b706fdd57a954ee9da2d0ff97b02d",
|
|
||||||
"branch": "master",
|
|
||||||
"notests": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"importpath": "github.com/apex/log/handlers/discard",
|
|
||||||
"repository": "https://github.com/apex/log",
|
|
||||||
"vcs": "git",
|
|
||||||
"revision": "c20bfcdb771b706fdd57a954ee9da2d0ff97b02d",
|
|
||||||
"branch": "master",
|
|
||||||
"path": "/handlers/discard",
|
|
||||||
"notests": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"importpath": "github.com/aws/aws-sdk-go/aws",
|
|
||||||
"repository": "https://github.com/aws/aws-sdk-go",
|
|
||||||
"vcs": "git",
|
|
||||||
"revision": "966f1a968ba8e260628dfa6be07bcd48d8e98ef6",
|
|
||||||
"branch": "master",
|
|
||||||
"path": "/aws",
|
|
||||||
"notests": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"importpath": "github.com/aws/aws-sdk-go/aws/session",
|
|
||||||
"repository": "https://github.com/aws/aws-sdk-go",
|
|
||||||
"vcs": "git",
|
|
||||||
"revision": "966f1a968ba8e260628dfa6be07bcd48d8e98ef6",
|
|
||||||
"branch": "master",
|
|
||||||
"path": "/aws/session",
|
|
||||||
"notests": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"importpath": "github.com/aws/aws-sdk-go/private/endpoints",
|
|
||||||
"repository": "https://github.com/aws/aws-sdk-go",
|
|
||||||
"vcs": "git",
|
|
||||||
"revision": "966f1a968ba8e260628dfa6be07bcd48d8e98ef6",
|
|
||||||
"branch": "master",
|
|
||||||
"path": "/private/endpoints",
|
|
||||||
"notests": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"importpath": "github.com/aws/aws-sdk-go/private/protocol",
|
|
||||||
"repository": "https://github.com/aws/aws-sdk-go",
|
|
||||||
"vcs": "git",
|
|
||||||
"revision": "966f1a968ba8e260628dfa6be07bcd48d8e98ef6",
|
|
||||||
"branch": "master",
|
|
||||||
"path": "/private/protocol",
|
|
||||||
"notests": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"importpath": "github.com/aws/aws-sdk-go/private/signer/v4",
|
|
||||||
"repository": "https://github.com/aws/aws-sdk-go",
|
|
||||||
"vcs": "git",
|
|
||||||
"revision": "966f1a968ba8e260628dfa6be07bcd48d8e98ef6",
|
|
||||||
"branch": "master",
|
|
||||||
"path": "/private/signer/v4",
|
|
||||||
"notests": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"importpath": "github.com/aws/aws-sdk-go/private/waiter",
|
|
||||||
"repository": "https://github.com/aws/aws-sdk-go",
|
|
||||||
"vcs": "git",
|
|
||||||
"revision": "966f1a968ba8e260628dfa6be07bcd48d8e98ef6",
|
|
||||||
"branch": "master",
|
|
||||||
"path": "/private/waiter",
|
|
||||||
"notests": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"importpath": "github.com/aws/aws-sdk-go/service/kinesis",
|
|
||||||
"repository": "https://github.com/aws/aws-sdk-go",
|
|
||||||
"vcs": "git",
|
|
||||||
"revision": "966f1a968ba8e260628dfa6be07bcd48d8e98ef6",
|
|
||||||
"branch": "master",
|
|
||||||
"path": "/service/kinesis",
|
|
||||||
"notests": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"importpath": "github.com/go-ini/ini",
|
|
||||||
"repository": "https://github.com/go-ini/ini",
|
|
||||||
"vcs": "git",
|
|
||||||
"revision": "12f418cc7edc5a618a51407b7ac1f1f512139df3",
|
|
||||||
"branch": "master",
|
|
||||||
"notests": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"importpath": "github.com/go-logfmt/logfmt",
|
|
||||||
"repository": "https://github.com/go-logfmt/logfmt",
|
|
||||||
"vcs": "git",
|
|
||||||
"revision": "08ab82a63ef462ac643ec79e659f023891f588f5",
|
|
||||||
"branch": "master",
|
|
||||||
"notests": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"importpath": "github.com/hoisie/redis",
|
|
||||||
"repository": "https://github.com/hoisie/redis",
|
|
||||||
"vcs": "git",
|
|
||||||
"revision": "788f01e396a99c96c8f56338383926f16841ebae",
|
|
||||||
"branch": "master",
|
|
||||||
"notests": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"importpath": "github.com/jmespath/go-jmespath",
|
|
||||||
"repository": "https://github.com/jmespath/go-jmespath",
|
|
||||||
"vcs": "git",
|
|
||||||
"revision": "0b12d6b521d83fc7f755e7cfc1b1fbdd35a01a74",
|
|
||||||
"branch": "master",
|
|
||||||
"notests": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"importpath": "github.com/jpillora/backoff",
|
|
||||||
"repository": "https://github.com/jpillora/backoff",
|
|
||||||
"vcs": "git",
|
|
||||||
"revision": "0496a6c14df020789376f4d4a261273d5ddb36ec",
|
|
||||||
"branch": "master",
|
|
||||||
"notests": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"importpath": "github.com/kr/pretty",
|
|
||||||
"repository": "https://github.com/kr/pretty",
|
|
||||||
"vcs": "git",
|
|
||||||
"revision": "add1dbc86daf0f983cd4a48ceb39deb95c729b67",
|
|
||||||
"branch": "master",
|
|
||||||
"notests": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"importpath": "github.com/kr/text",
|
|
||||||
"repository": "https://github.com/kr/text",
|
|
||||||
"vcs": "git",
|
|
||||||
"revision": "bb797dc4fb8320488f47bf11de07a733d7233e1f",
|
|
||||||
"branch": "master",
|
|
||||||
"notests": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"importpath": "github.com/rogpeppe/fastuuid",
|
|
||||||
"repository": "https://github.com/rogpeppe/fastuuid",
|
|
||||||
"vcs": "git",
|
|
||||||
"revision": "6724a57986aff9bff1a1770e9347036def7c89f6",
|
|
||||||
"branch": "master",
|
|
||||||
"notests": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"importpath": "github.com/tj/go-elastic/batch",
|
|
||||||
"repository": "https://github.com/tj/go-elastic",
|
|
||||||
"vcs": "git",
|
|
||||||
"revision": "6923cdab333be878d768a2b563cd25bc816c3dab",
|
|
||||||
"branch": "master",
|
|
||||||
"path": "/batch",
|
|
||||||
"notests": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"importpath": "github.com/tj/go-kinesis",
|
|
||||||
"repository": "https://github.com/tj/go-kinesis",
|
|
||||||
"vcs": "git",
|
|
||||||
"revision": "677d118cb62c634d6d8dc7c49e979c4d40fac58b",
|
|
||||||
"branch": "master",
|
|
||||||
"notests": true
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
309
vendor/vendor.json
vendored
Normal file
309
vendor/vendor.json
vendored
Normal file
|
|
@ -0,0 +1,309 @@
|
||||||
|
{
|
||||||
|
"comment": "",
|
||||||
|
"ignore": "test",
|
||||||
|
"package": [
|
||||||
|
{
|
||||||
|
"checksumSHA1": "Ur88QI//9Ue82g83qvBSakGlzVg=",
|
||||||
|
"path": "github.com/apex/log",
|
||||||
|
"revision": "4ea85e918cc8389903d5f12d7ccac5c23ab7d89b",
|
||||||
|
"revisionTime": "2016-09-05T15:13:04Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "o5a5xWoaGDKEnNy0W7TikB66lMc=",
|
||||||
|
"path": "github.com/apex/log/handlers/text",
|
||||||
|
"revision": "4ea85e918cc8389903d5f12d7ccac5c23ab7d89b",
|
||||||
|
"revisionTime": "2016-09-05T15:13:04Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "dSo0vFXJGuTtd6H80q8ZczLszJM=",
|
||||||
|
"path": "github.com/aws/aws-sdk-go/aws",
|
||||||
|
"revision": "f34b74c96bfd27df35643adeb14d8431ca047df5",
|
||||||
|
"revisionTime": "2016-08-17T18:35:19Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "Y9W+4GimK4Fuxq+vyIskVYFRnX4=",
|
||||||
|
"path": "github.com/aws/aws-sdk-go/aws/awserr",
|
||||||
|
"revision": "f34b74c96bfd27df35643adeb14d8431ca047df5",
|
||||||
|
"revisionTime": "2016-08-17T18:35:19Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "dkfyy7aRNZ6BmUZ4ZdLIcMMXiPA=",
|
||||||
|
"path": "github.com/aws/aws-sdk-go/aws/awsutil",
|
||||||
|
"revision": "f34b74c96bfd27df35643adeb14d8431ca047df5",
|
||||||
|
"revisionTime": "2016-08-17T18:35:19Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "H/tMKHZU+Qka6RtYiGB50s2uA0s=",
|
||||||
|
"path": "github.com/aws/aws-sdk-go/aws/client",
|
||||||
|
"revision": "f34b74c96bfd27df35643adeb14d8431ca047df5",
|
||||||
|
"revisionTime": "2016-08-17T18:35:19Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "ieAJ+Cvp/PKv1LpUEnUXpc3OI6E=",
|
||||||
|
"path": "github.com/aws/aws-sdk-go/aws/client/metadata",
|
||||||
|
"revision": "f34b74c96bfd27df35643adeb14d8431ca047df5",
|
||||||
|
"revisionTime": "2016-08-17T18:35:19Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "gNWirlrTfSLbOe421hISBAhTqa4=",
|
||||||
|
"path": "github.com/aws/aws-sdk-go/aws/corehandlers",
|
||||||
|
"revision": "f34b74c96bfd27df35643adeb14d8431ca047df5",
|
||||||
|
"revisionTime": "2016-08-17T18:35:19Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "dNZNaOPfBPnzE2CBnfhXXZ9g9jU=",
|
||||||
|
"path": "github.com/aws/aws-sdk-go/aws/credentials",
|
||||||
|
"revision": "f34b74c96bfd27df35643adeb14d8431ca047df5",
|
||||||
|
"revisionTime": "2016-08-17T18:35:19Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "KQiUK/zr3mqnAXD7x/X55/iNme0=",
|
||||||
|
"path": "github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds",
|
||||||
|
"revision": "f34b74c96bfd27df35643adeb14d8431ca047df5",
|
||||||
|
"revisionTime": "2016-08-17T18:35:19Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "NUJUTWlc1sV8b7WjfiYc4JZbXl0=",
|
||||||
|
"path": "github.com/aws/aws-sdk-go/aws/credentials/endpointcreds",
|
||||||
|
"revision": "f34b74c96bfd27df35643adeb14d8431ca047df5",
|
||||||
|
"revisionTime": "2016-08-17T18:35:19Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "4Ipx+5xN0gso+cENC2MHMWmQlR4=",
|
||||||
|
"path": "github.com/aws/aws-sdk-go/aws/credentials/stscreds",
|
||||||
|
"revision": "f34b74c96bfd27df35643adeb14d8431ca047df5",
|
||||||
|
"revisionTime": "2016-08-17T18:35:19Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "nCMd1XKjgV21bEl7J8VZFqTV8PE=",
|
||||||
|
"path": "github.com/aws/aws-sdk-go/aws/defaults",
|
||||||
|
"revision": "f34b74c96bfd27df35643adeb14d8431ca047df5",
|
||||||
|
"revisionTime": "2016-08-17T18:35:19Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "U0SthWum+t9ACanK7SDJOg3dO6M=",
|
||||||
|
"path": "github.com/aws/aws-sdk-go/aws/ec2metadata",
|
||||||
|
"revision": "f34b74c96bfd27df35643adeb14d8431ca047df5",
|
||||||
|
"revisionTime": "2016-08-17T18:35:19Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "NyUg1P8ZS/LHAAQAk/4C5O4X3og=",
|
||||||
|
"path": "github.com/aws/aws-sdk-go/aws/request",
|
||||||
|
"revision": "f34b74c96bfd27df35643adeb14d8431ca047df5",
|
||||||
|
"revisionTime": "2016-08-17T18:35:19Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "tBdFneml1Vn7uvezcktsa+hUsGg=",
|
||||||
|
"path": "github.com/aws/aws-sdk-go/aws/session",
|
||||||
|
"revision": "f34b74c96bfd27df35643adeb14d8431ca047df5",
|
||||||
|
"revisionTime": "2016-08-17T18:35:19Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "7lla+sckQeF18wORAGuU2fFMlp4=",
|
||||||
|
"path": "github.com/aws/aws-sdk-go/aws/signer/v4",
|
||||||
|
"revision": "f34b74c96bfd27df35643adeb14d8431ca047df5",
|
||||||
|
"revisionTime": "2016-08-17T18:35:19Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "Bm6UrYb2QCzpYseLwwgw6aetgRc=",
|
||||||
|
"path": "github.com/aws/aws-sdk-go/private/endpoints",
|
||||||
|
"revision": "f34b74c96bfd27df35643adeb14d8431ca047df5",
|
||||||
|
"revisionTime": "2016-08-17T18:35:19Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "wk7EyvDaHwb5qqoOP/4d3cV0708=",
|
||||||
|
"path": "github.com/aws/aws-sdk-go/private/protocol",
|
||||||
|
"revision": "f34b74c96bfd27df35643adeb14d8431ca047df5",
|
||||||
|
"revisionTime": "2016-08-17T18:35:19Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "L7xWYwx0jNQnzlYHwBS+1q6DcCI=",
|
||||||
|
"path": "github.com/aws/aws-sdk-go/private/protocol/json/jsonutil",
|
||||||
|
"revision": "f34b74c96bfd27df35643adeb14d8431ca047df5",
|
||||||
|
"revisionTime": "2016-08-17T18:35:19Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "H9TymcQkQnXSXSVfjggiiS4bpzM=",
|
||||||
|
"path": "github.com/aws/aws-sdk-go/private/protocol/jsonrpc",
|
||||||
|
"revision": "f34b74c96bfd27df35643adeb14d8431ca047df5",
|
||||||
|
"revisionTime": "2016-08-17T18:35:19Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "isoix7lTx4qIq2zI2xFADtti5SI=",
|
||||||
|
"path": "github.com/aws/aws-sdk-go/private/protocol/query",
|
||||||
|
"revision": "f34b74c96bfd27df35643adeb14d8431ca047df5",
|
||||||
|
"revisionTime": "2016-08-17T18:35:19Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "5xzix1R8prUyWxgLnzUQoxTsfik=",
|
||||||
|
"path": "github.com/aws/aws-sdk-go/private/protocol/query/queryutil",
|
||||||
|
"revision": "f34b74c96bfd27df35643adeb14d8431ca047df5",
|
||||||
|
"revisionTime": "2016-08-17T18:35:19Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "TW/7U+/8ormL7acf6z2rv2hDD+s=",
|
||||||
|
"path": "github.com/aws/aws-sdk-go/private/protocol/rest",
|
||||||
|
"revision": "f34b74c96bfd27df35643adeb14d8431ca047df5",
|
||||||
|
"revisionTime": "2016-08-17T18:35:19Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "Y6Db2GGfGD9LPpcJIPj8vXE8BbQ=",
|
||||||
|
"path": "github.com/aws/aws-sdk-go/private/protocol/restxml",
|
||||||
|
"revision": "f34b74c96bfd27df35643adeb14d8431ca047df5",
|
||||||
|
"revisionTime": "2016-08-17T18:35:19Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "eUEkjyMPAuekKBE4ou+nM9tXEas=",
|
||||||
|
"path": "github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil",
|
||||||
|
"revision": "f34b74c96bfd27df35643adeb14d8431ca047df5",
|
||||||
|
"revisionTime": "2016-08-17T18:35:19Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "Eo9yODN5U99BK0pMzoqnBm7PCrY=",
|
||||||
|
"path": "github.com/aws/aws-sdk-go/private/waiter",
|
||||||
|
"revision": "f34b74c96bfd27df35643adeb14d8431ca047df5",
|
||||||
|
"revisionTime": "2016-08-17T18:35:19Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "TtIAgZ+evpkKB5bBYCB69k0wZoU=",
|
||||||
|
"path": "github.com/aws/aws-sdk-go/service/firehose",
|
||||||
|
"revision": "f34b74c96bfd27df35643adeb14d8431ca047df5",
|
||||||
|
"revisionTime": "2016-08-17T18:35:19Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "2n5/m0ClE4OyQRNdjfLwg+nSY3o=",
|
||||||
|
"path": "github.com/aws/aws-sdk-go/service/kinesis",
|
||||||
|
"revision": "f34b74c96bfd27df35643adeb14d8431ca047df5",
|
||||||
|
"revisionTime": "2016-08-17T18:35:19Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "fgZ1cdh2T0cWRorIZkMGFDADMQw=",
|
||||||
|
"path": "github.com/aws/aws-sdk-go/service/kinesis/kinesisiface",
|
||||||
|
"revision": "f34b74c96bfd27df35643adeb14d8431ca047df5",
|
||||||
|
"revisionTime": "2016-08-17T18:35:19Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "imxJucuPrgaPRMPtAgsu+Y7soB4=",
|
||||||
|
"path": "github.com/aws/aws-sdk-go/service/s3",
|
||||||
|
"revision": "f34b74c96bfd27df35643adeb14d8431ca047df5",
|
||||||
|
"revisionTime": "2016-08-17T18:35:19Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "nH/itbdeFHpl4ysegdtgww9bFSA=",
|
||||||
|
"path": "github.com/aws/aws-sdk-go/service/sts",
|
||||||
|
"revision": "f34b74c96bfd27df35643adeb14d8431ca047df5",
|
||||||
|
"revisionTime": "2016-08-17T18:35:19Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "zUyQXVCSaV97bbiVZbX1qn8UWm4=",
|
||||||
|
"path": "github.com/bmizerany/assert",
|
||||||
|
"revision": "e17e99893cb6509f428e1728281c2ad60a6b31e3",
|
||||||
|
"revisionTime": "2012-07-16T20:56:30Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "i7BD7wKsIrix92VtlJ4zQRP4G8c=",
|
||||||
|
"path": "github.com/crowdmob/goamz/aws",
|
||||||
|
"revision": "3a06871fe9fc0281ca90f3a7d97258d042ed64c0",
|
||||||
|
"revisionTime": "2015-01-28T19:49:25Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "qijq0UWIx8EKPT+GbsbuaZMw/gA=",
|
||||||
|
"path": "github.com/crowdmob/goamz/s3",
|
||||||
|
"revision": "3a06871fe9fc0281ca90f3a7d97258d042ed64c0",
|
||||||
|
"revisionTime": "2015-01-28T19:49:25Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "FCeEm2BWZV/n4oTy+SGd/k0Ab5c=",
|
||||||
|
"origin": "github.com/aws/aws-sdk-go/vendor/github.com/go-ini/ini",
|
||||||
|
"path": "github.com/go-ini/ini",
|
||||||
|
"revision": "f34b74c96bfd27df35643adeb14d8431ca047df5",
|
||||||
|
"revisionTime": "2016-08-17T18:35:19Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "0ZrwvB6KoGPj2PoDNSEJwxQ6Mog=",
|
||||||
|
"origin": "github.com/aws/aws-sdk-go/vendor/github.com/jmespath/go-jmespath",
|
||||||
|
"path": "github.com/jmespath/go-jmespath",
|
||||||
|
"revision": "f34b74c96bfd27df35643adeb14d8431ca047df5",
|
||||||
|
"revisionTime": "2016-08-17T18:35:19Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "IXVypCQsDOlWf8dqyFogbOsRdvM=",
|
||||||
|
"path": "github.com/jpillora/backoff",
|
||||||
|
"revision": "0496a6c14df020789376f4d4a261273d5ddb36ec",
|
||||||
|
"revisionTime": "2016-04-14T05:52:04Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "1YeGotQXMZMqk+mmm8sbBVJywpw=",
|
||||||
|
"path": "github.com/kr/pretty",
|
||||||
|
"revision": "e6ac2fc51e89a3249e82157fa0bb7a18ef9dd5bb",
|
||||||
|
"revisionTime": "2015-05-20T16:35:14Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "uulQHQ7IsRKqDudBC8Go9J0gtAc=",
|
||||||
|
"path": "github.com/kr/text",
|
||||||
|
"revision": "bb797dc4fb8320488f47bf11de07a733d7233e1f",
|
||||||
|
"revisionTime": "2015-09-05T22:45:08Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "Tivm2ueYu71B9YxTEyGxe+8ZWgk=",
|
||||||
|
"path": "github.com/lib/pq",
|
||||||
|
"revision": "f59175c2986495ff94109dee3835c504a96c3e81",
|
||||||
|
"revisionTime": "2016-01-27T22:38:42Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "xppHi82MLqVx1eyQmbhTesAEjx8=",
|
||||||
|
"path": "github.com/lib/pq/oid",
|
||||||
|
"revision": "f59175c2986495ff94109dee3835c504a96c3e81",
|
||||||
|
"revisionTime": "2016-01-27T22:38:42Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "QI1tJqI+jMmFrCAKcXs+LefgES4=",
|
||||||
|
"path": "github.com/tj/go-kinesis",
|
||||||
|
"revision": "817ff40136c6d4909bcff1021e58fdedf788ba23",
|
||||||
|
"revisionTime": "2016-06-02T03:00:41Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "+4r0PnLwwyhO5/jvU5R/TEJb4kA=",
|
||||||
|
"path": "gopkg.in/bsm/ratelimit.v1",
|
||||||
|
"revision": "db14e161995a5177acef654cb0dd785e8ee8bc22",
|
||||||
|
"revisionTime": "2016-02-20T15:49:07Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "JtXTQXRlxRB///NYmPDuMpEpvNI=",
|
||||||
|
"path": "gopkg.in/redis.v5",
|
||||||
|
"revision": "854c88a72c8bb9c09936145aef886b7697d6b995",
|
||||||
|
"revisionTime": "2016-12-03T15:45:52Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "vQSE4FOH4EvyzYA72w60XOetmVY=",
|
||||||
|
"path": "gopkg.in/redis.v5/internal",
|
||||||
|
"revision": "854c88a72c8bb9c09936145aef886b7697d6b995",
|
||||||
|
"revisionTime": "2016-12-03T15:45:52Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "2Ek4SixeRSKOX3mUiBMs3Aw+Guc=",
|
||||||
|
"path": "gopkg.in/redis.v5/internal/consistenthash",
|
||||||
|
"revision": "854c88a72c8bb9c09936145aef886b7697d6b995",
|
||||||
|
"revisionTime": "2016-12-03T15:45:52Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "rJYVKcBrwYUGl7nuuusmZGrt8mY=",
|
||||||
|
"path": "gopkg.in/redis.v5/internal/hashtag",
|
||||||
|
"revision": "854c88a72c8bb9c09936145aef886b7697d6b995",
|
||||||
|
"revisionTime": "2016-12-03T15:45:52Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "VnsHRPAMRMuhz7/n/85MZwMrchQ=",
|
||||||
|
"path": "gopkg.in/redis.v5/internal/pool",
|
||||||
|
"revision": "854c88a72c8bb9c09936145aef886b7697d6b995",
|
||||||
|
"revisionTime": "2016-12-03T15:45:52Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "604uyPTNWLBNAnAyNRMiwYHXknA=",
|
||||||
|
"path": "gopkg.in/redis.v5/internal/proto",
|
||||||
|
"revision": "854c88a72c8bb9c09936145aef886b7697d6b995",
|
||||||
|
"revisionTime": "2016-12-03T15:45:52Z"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"rootPath": "github.com/harlow/kinesis-connectors"
|
||||||
|
}
|
||||||
Loading…
Reference in a new issue