Add dummy users data for producing onto stream
Pulled from: https://github.com/awslabs/amazon-kinesis-connectors
This commit is contained in:
parent
4f374e4425
commit
2f58b136fe
2 changed files with 5007 additions and 11 deletions
|
|
@ -13,23 +13,19 @@ import (
|
||||||
"github.com/aws/aws-sdk-go/service/kinesis"
|
"github.com/aws/aws-sdk-go/service/kinesis"
|
||||||
)
|
)
|
||||||
|
|
||||||
var svc = kinesis.New(session.New(), &aws.Config{
|
|
||||||
Region: aws.String("us-west-1"),
|
|
||||||
})
|
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
var streamName = flag.String("stream", "", "Stream name")
|
var streamName = flag.String("stream", "", "Stream name")
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
|
||||||
// download file with test data
|
// open dummy user data
|
||||||
// curl https://s3.amazonaws.com/kinesis.test/users.txt -o /tmp/users.txt
|
f, err := os.Open("users.txt")
|
||||||
f, err := os.Open("/tmp/users.txt")
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal("Cannot open users.txt file")
|
log.Fatal("Cannot open users.txt file")
|
||||||
}
|
}
|
||||||
defer f.Close()
|
defer f.Close()
|
||||||
|
|
||||||
var records []*kinesis.PutRecordsRequestEntry
|
var records []*kinesis.PutRecordsRequestEntry
|
||||||
|
var client = kinesis.New(session.New())
|
||||||
|
|
||||||
// loop over file data
|
// loop over file data
|
||||||
b := bufio.NewScanner(f)
|
b := bufio.NewScanner(f)
|
||||||
|
|
@ -40,18 +36,18 @@ func main() {
|
||||||
})
|
})
|
||||||
|
|
||||||
if len(records) > 250 {
|
if len(records) > 250 {
|
||||||
putRecords(streamName, records)
|
putRecords(client, streamName, records)
|
||||||
records = nil
|
records = nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(records) > 0 {
|
if len(records) > 0 {
|
||||||
putRecords(streamName, records)
|
putRecords(client, streamName, records)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func putRecords(streamName *string, records []*kinesis.PutRecordsRequestEntry) {
|
func putRecords(client *kinesis.Kinesis, streamName *string, records []*kinesis.PutRecordsRequestEntry) {
|
||||||
_, err := svc.PutRecords(&kinesis.PutRecordsInput{
|
_, err := client.PutRecords(&kinesis.PutRecordsInput{
|
||||||
StreamName: streamName,
|
StreamName: streamName,
|
||||||
Records: records,
|
Records: records,
|
||||||
})
|
})
|
||||||
|
|
|
||||||
5000
examples/producer/users.txt
Normal file
5000
examples/producer/users.txt
Normal file
File diff suppressed because it is too large
Load diff
Loading…
Reference in a new issue