57dc74bd48
* Add first checkpoint implementation of RedisCheckpoint. * Move MsgBuffer into the buffers.go file.
37 lines
675 B
Go
37 lines
675 B
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"github.com/harlow/go-etl"
|
|
"github.com/joho/godotenv"
|
|
"github.com/sendgridlabs/go-kinesis"
|
|
)
|
|
|
|
func main() {
|
|
godotenv.Load()
|
|
|
|
k := kinesis.New("", "", kinesis.Region{})
|
|
s := "inputStream"
|
|
|
|
c := etl.RedisCheckpoint{appName: "sampleApp"}
|
|
|
|
e := etl.S3Emitter{}
|
|
e.SetBucketName("bucketName")
|
|
|
|
// t := etl.EventTransformer{}
|
|
|
|
args := kinesis.NewArgs()
|
|
args.Add("StreamName", s)
|
|
streamInfo, err := k.DescribeStream(args)
|
|
|
|
if err != nil {
|
|
fmt.Printf("Unable to connect to %v stream. Aborting.", s)
|
|
return
|
|
}
|
|
|
|
for _, shard := range streamInfo.StreamDescription.Shards {
|
|
go etl.GetRecords(k, &c, e, s, shard.ShardId)
|
|
}
|
|
|
|
select {}
|
|
}
|