25e390e8b4
* Reduce the size of main README in favor of adding functioning examples * Add S3 Pipeline exmaple * Add example of seeing the stream
54 lines
944 B
Go
54 lines
944 B
Go
package main
|
|
|
|
import (
|
|
"bufio"
|
|
"fmt"
|
|
"os"
|
|
"sync"
|
|
|
|
"github.com/harlow/kinesis-connectors"
|
|
"github.com/joho/godotenv"
|
|
"github.com/sendgridlabs/go-kinesis"
|
|
)
|
|
|
|
func main() {
|
|
godotenv.Load()
|
|
|
|
// Initialize Kinesis client
|
|
auth := kinesis.NewAuth()
|
|
ksis := kinesis.New(&auth, kinesis.Region{})
|
|
|
|
// Create stream
|
|
connector.CreateStream(ksis, "userStream", 2)
|
|
|
|
// read file
|
|
file, _ := os.Open("users.txt")
|
|
defer file.Close()
|
|
scanner := bufio.NewScanner(file)
|
|
|
|
args := kinesis.NewArgs()
|
|
args.Add("StreamName", "userStream")
|
|
ctr := 0
|
|
var wg sync.WaitGroup
|
|
|
|
for scanner.Scan() {
|
|
l := scanner.Text()
|
|
ctr = ctr + 1
|
|
key := fmt.Sprintf("partitionKey-%d", ctr)
|
|
|
|
args := kinesis.NewArgs()
|
|
args.Add("StreamName", "userStream")
|
|
args.AddRecord([]byte(l), key)
|
|
wg.Add(1)
|
|
|
|
go func() {
|
|
ksis.PutRecords(args)
|
|
fmt.Print(".")
|
|
wg.Done()
|
|
}()
|
|
}
|
|
|
|
wg.Wait()
|
|
fmt.Println(".")
|
|
fmt.Println("Finished populating userStream")
|
|
}
|