add test for default case

batchconsumer/writer.go

@@ -58,7 +58,7 @@ func (b *batchedWriter) Initialize(shardID string, checkpointer kcl.Checkpointer
 func (b *batchedWriter) splitMessageIfNecessary(record []byte) ([][]byte, error) {
 	// We handle two types of records:
 	// - records emitted from CWLogs Subscription
-	// - records emiited from KPL
+	// - records emitted from KPL
 	if !splitter.IsGzipped(record) {
 		// Process a single message, from KPL
 		return [][]byte{record}, nil

golang.mk

@@ -1,7 +1,7 @@
 # This is the default Clever Golang Makefile.
 # It is stored in the dev-handbook repo, github.com/Clever/dev-handbook
 # Please do not alter this file directly.
-GOLANG_MK_VERSION := 0.2.0
+GOLANG_MK_VERSION := 0.3.2
 
 SHELL := /bin/bash
 SYSTEM := $(shell uname -a | cut -d" " -f1 | tr '[:upper:]' '[:lower:]')
@@ -18,7 +18,7 @@ define golang-version-check
 _ := $(if  \
 		$(shell  \
 			expr >/dev/null  \
-				`go version | cut -d" " -f3 | cut -c3- | cut -d. -f2`  \
+				`go version | cut -d" " -f3 | cut -c3- | cut -d. -f2 | sed -E 's/beta[0-9]+//'`  \
 				\>= `echo $(1) | cut -d. -f2`  \
 				\&  \
 				`go version | cut -d" " -f3 | cut -c3- | cut -d. -f1`  \
@@ -52,9 +52,16 @@ golang-dep-vendor-deps: bin/dep
 # golang-godep-vendor is a target for saving dependencies with the dep tool
 # to the vendor/ directory. All nested vendor/ directories are deleted via
 # the prune command.
+# In CI, -vendor-only is used to avoid updating the lock file.
+ifndef CI
 define golang-dep-vendor
-bin/dep ensure
+bin/dep ensure -v
 endef
+else
+define golang-dep-vendor
+bin/dep ensure -v -vendor-only
+endef
+endif
 
 # Golint is a tool for linting Golang code for common errors.
 GOLINT := $(GOPATH)/bin/golint
@@ -147,6 +154,19 @@ $(call golang-vet,$(1))
 $(call golang-test-strict,$(1))
 endef
 
+# golang-build: builds a golang binary. ensures CGO build is done during CI. This is needed to make a binary that works with a Docker alpine image.
+# arg1: pkg path
+# arg2: executable name
+define golang-build
+@echo "BUILDING..."
+@if [ -z "$$CI" ]; then \
+	go build -o bin/$(2) $(1); \
+else \
+	echo "-> Building CGO binary"; \
+	CGO_ENABLED=0 go build -installsuffix cgo -o bin/$(2) $(1); \
+fi;
+endef
+
 # golang-update-makefile downloads latest version of golang.mk
 golang-update-makefile:
 	@wget https://raw.githubusercontent.com/Clever/dev-handbook/master/make/golang.mk -O /tmp/golang.mk 2>/dev/null

splitter/splitter.go

@@ -108,8 +108,9 @@ var awsBatchTaskRegex = regexp.MustCompile(awsBatchTaskMeta)
 var awsLambdaLogGroupRegex = regexp.MustCompile(`^/aws/lambda/([a-z0-9-]+)--([a-z0-9-]+)$`)
 var awsLambdaRequestIDRegex = regexp.MustCompile(`[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}`)
 
-// arn cruft to satisfy parsing later on: https://github.com/Clever/amazon-kinesis-client-go/blob/94aacdf8339bd2cc8400d3bcb323dc1bce2c8422/decode/decode.go#L421-L425
+// arn and task cruft to satisfy parsing later on: https://github.com/Clever/amazon-kinesis-client-go/blob/94aacdf8339bd2cc8400d3bcb323dc1bce2c8422/decode/decode.go#L421-L425
 const arnCruft = `/arn%3Aaws%3Aecs%3Aus-east-1%3A999988887777%3Atask%2F`
+const taskCruft = `12345678-1234-1234-1234-555566667777`
 
 type RSysLogMessage struct {
 	Timestamp   time.Time
@@ -180,8 +181,8 @@ func splitDefault(b LogEventBatch) []RSysLogMessage {
 	for _, event := range b.LogEvents {
 		out = append(out, RSysLogMessage{
 			Timestamp:   event.Timestamp.Time(),
-			Hostname:    "unknown",
-			ProgramName: "unknown",
+			Hostname:    b.LogStream,
+			ProgramName: b.LogGroup + "--" + b.LogStream + arnCruft + taskCruft,
 			PID:         1,
 			Message:     event.Message,
 		})

splitter/splitter_test.go

@@ -153,3 +153,25 @@ func TestSplitLambda(t *testing.T) {
 	}
 	assert.Equal(t, expected, lines)
 }
+
+func TestSplitDefault(t *testing.T) {
+	input := LogEventBatch{
+		MessageType:         "DATA_MESSAGE",
+		Owner:               "123456789012",
+		LogGroup:            "vpn_flow_logs",
+		LogStream:           "eni-43403819-all",
+		SubscriptionFilters: []string{"SomeSubscription"},
+		LogEvents: []LogEvent{
+			{
+				ID:        "99999992379011144044923130086453437181614530551221780480",
+				Timestamp: NewUnixTimestampMillis(1498519943285),
+				Message:   "2 589690932525 eni-43403819 10.0.0.233 172.217.6.46 64067 443 17 8 3969 1516891809 1516891868 ACCEPT OK",
+			},
+		},
+	}
+	lines := Split(input)
+	expected := [][]byte{
+		[]byte(`2017-06-26T23:32:23.285001+00:00 eni-43403819-all vpn_flow_logs--eni-43403819-all/arn%3Aaws%3Aecs%3Aus-east-1%3A999988887777%3Atask%2F12345678-1234-1234-1234-555566667777[1]: 2 589690932525 eni-43403819 10.0.0.233 172.217.6.46 64067 443 17 8 3969 1516891809 1516891868 ACCEPT OK`),
+	}
+	assert.Equal(t, expected, lines)
+}