Reworded docs to remove references to firehose. Renamed config variables to be clearer.

batchconsumer/consumer.go

@@ -18,12 +18,12 @@ type Config struct {
 
 	// LogFile where consumer errors and failed log lines are saved
 	LogFile string
-	// FlushInterval is how often accumulated messages should be bulk put to firehose
+	// BatchInterval the upper bound on how often SendBatch is called with accumulated messages
-	FlushInterval time.Duration
+	BatchInterval time.Duration
-	// FlushCount is the number of messages that triggers a push to firehose. Max batch size is 500, see: http://docs.aws.amazon.com/firehose/latest/dev/limits.html
+	// BatchCount is the number of messages that triggers a SendBatch call
-	FlushCount int
+	BatchCount int
-	// FlushSize is the size of a batch in bytes that triggers a push to firehose. Max batch size is 4Mb (4*1024*1024), see: http://docs.aws.amazon.com/firehose/latest/dev/limits.html
+	// BatchSize is the size of a batch in bytes that triggers a SendBatch call
-	FlushSize int
+	BatchSize int
 
 	// ReadRateLimit the number of records read per seconds
 	ReadRateLimit int
@@ -49,14 +49,14 @@ func withDefaults(config Config) Config {
 		config.LogFile = "/tmp/kcl-" + time.Now().Format(time.RFC3339)
 	}
 
-	if config.FlushInterval == 0 {
+	if config.BatchInterval == 0 {
-		config.FlushInterval = 10 * time.Second
+		config.BatchInterval = 10 * time.Second
 	}
-	if config.FlushCount == 0 {
+	if config.BatchCount == 0 {
-		config.FlushCount = 500
+		config.BatchCount = 500
 	}
-	if config.FlushSize == 0 {
+	if config.BatchSize == 0 {
-		config.FlushSize = 4 * 1024 * 1024
+		config.BatchSize = 4 * 1024 * 1024
 	}
 
 	if config.DeployEnv == "" {

batchconsumer/writer.go

@@ -118,7 +118,7 @@ func (b *batchedWriter) createBatcher(tag string) batcher.Batcher {
 		tag:    tag,
 		writer: b,
 	}
-	return batcher.New(sync, b.config.FlushInterval, b.config.FlushCount, b.config.FlushSize)
+	return batcher.New(sync, b.config.BatchInterval, b.config.BatchCount, b.config.BatchSize)
 }
 
 func (b *batchedWriter) splitMessageIfNecessary(record []byte) ([][]byte, error) {

cmd/batchconsumer/main.go

@@ -25,9 +25,9 @@ func createDummyOutput() (logger.KayveeLogger, *os.File) {
 
 func main() {
 	config := kbc.Config{
-		FlushInterval: 10 * time.Second,
+		BatchInterval: 10 * time.Second,
-		FlushCount:    500,
+		BatchCount:    500,
-		FlushSize:     4 * 1024 * 1024, // 4Mb
+		BatchSize:     4 * 1024 * 1024, // 4Mb
 		LogFile:       "/tmp/example-kcl-consumer",
 		DeployEnv:     "test-env",
 	}