kinesis-consumer/s3_emitter.go

package connector

import (
	"io"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"gopkg.in/matryer/try.v1"
)

// S3Emitter stores data in S3 bucket.
//
// The use of  this struct requires the configuration of an S3 bucket/endpoint. When the buffer is full, this
// struct's Emit method adds the contents of the buffer to S3 as one file. The filename is generated
// from the first and last sequence numbers of the records contained in that file separated by a
// dash. This struct requires the configuration of an S3 bucket and endpoint.
type S3Emitter struct {
	Bucket string
}

// Emit is invoked when the buffer is full. This method emits the set of filtered records.
func (e S3Emitter) Emit(s3Key string, b io.ReadSeeker) {
	svc := s3.New(session.New())
	params := &s3.PutObjectInput{
		Body:        b,
		Bucket:      aws.String(e.Bucket),
		ContentType: aws.String("text/plain"),
		Key:         aws.String(s3Key),
	}

	err := try.Do(func(attempt int) (bool, error) {
		var err error
		_, err = svc.PutObject(params)
		return attempt < 5, err
	})

	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok {
			logger.Log("error", "s3.PutObject", "code", awsErr.Code())
		}
	}

	logger.Log("info", "S3Emitter", "msg", "success", "key", s3Key)
}