fb98fbe244
Having an additional Client has added some confusion (https://github.com/harlow/kinesis-consumer/issues/45) on how to provide a custom kinesis client. Allowing `WithClient` to accept a Kinesis client it cleans up the interface. Major changes: * Remove the Client wrapper; prefer using kinesis client directly * Change `ScanError` to `ScanStatus` as the return value isn't necessarily an error Note: these are breaking changes, if you need last stable release please see here: https://github.com/harlow/kinesis-consumer/releases/tag/v0.2.0
62 lines
1.3 KiB
Go
62 lines
1.3 KiB
Go
package main
|
|
|
|
import (
|
|
"bufio"
|
|
"flag"
|
|
"fmt"
|
|
"log"
|
|
"os"
|
|
"time"
|
|
|
|
"github.com/aws/aws-sdk-go/aws"
|
|
"github.com/aws/aws-sdk-go/aws/session"
|
|
"github.com/aws/aws-sdk-go/service/kinesis"
|
|
)
|
|
|
|
var svc = kinesis.New(session.New(), &aws.Config{
|
|
Region: aws.String("us-west-1"),
|
|
})
|
|
|
|
func main() {
|
|
var streamName = flag.String("stream", "", "Stream name")
|
|
flag.Parse()
|
|
|
|
// download file with test data
|
|
// curl https://s3.amazonaws.com/kinesis.test/users.txt -o /tmp/users.txt
|
|
f, err := os.Open("/tmp/users.txt")
|
|
if err != nil {
|
|
log.Fatal("Cannot open users.txt file")
|
|
}
|
|
defer f.Close()
|
|
|
|
var records []*kinesis.PutRecordsRequestEntry
|
|
|
|
// loop over file data
|
|
b := bufio.NewScanner(f)
|
|
for b.Scan() {
|
|
records = append(records, &kinesis.PutRecordsRequestEntry{
|
|
Data: b.Bytes(),
|
|
PartitionKey: aws.String(time.Now().Format(time.RFC3339Nano)),
|
|
})
|
|
|
|
if len(records) > 250 {
|
|
putRecords(streamName, records)
|
|
records = nil
|
|
}
|
|
}
|
|
|
|
if len(records) > 0 {
|
|
putRecords(streamName, records)
|
|
}
|
|
}
|
|
|
|
func putRecords(streamName *string, records []*kinesis.PutRecordsRequestEntry) {
|
|
_, err := svc.PutRecords(&kinesis.PutRecordsInput{
|
|
StreamName: streamName,
|
|
Records: records,
|
|
})
|
|
if err != nil {
|
|
log.Fatalf("error putting records: %v", err)
|
|
}
|
|
fmt.Print(".")
|
|
}
|