f0e6461cb6
The previous pipeline model required a lot of setup and abstracted away the processing of records. By passing a HandlerFunc to the consumer we keep the business logic of processing of records closer to the use of the consumer. * Add refactoring note and SHA to README
38 lines
677 B
Go
38 lines
677 B
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"flag"
|
|
|
|
"github.com/harlow/kinesis-connectors"
|
|
)
|
|
|
|
var (
|
|
app = flag.String("a", "", "App name")
|
|
bucket = flag.String("b", "", "Bucket name")
|
|
stream = flag.String("s", "", "Stream name")
|
|
)
|
|
|
|
func handler(b connector.Buffer) {
|
|
body := new(bytes.Buffer)
|
|
|
|
// filter or transform data if needed
|
|
for _, r := range b.GetRecords() {
|
|
body.Write(r.Data)
|
|
}
|
|
|
|
s3 := &connector.S3Emitter{Bucket: *bucket}
|
|
s3.Emit(
|
|
connector.S3Key("", b.FirstSeq(), b.LastSeq()),
|
|
bytes.NewReader(body.Bytes()),
|
|
)
|
|
}
|
|
|
|
func main() {
|
|
flag.Parse()
|
|
|
|
c := connector.NewConsumer(*app, *stream)
|
|
c.Start(connector.HandlerFunc(handler))
|
|
|
|
select {} // run forever
|
|
}
|