diff --git a/Makefile b/Makefile index a075b95..7aa2bcf 100644 --- a/Makefile +++ b/Makefile @@ -62,4 +62,4 @@ $(PKGS): golang-test-all-deps install_deps: golang-dep-vendor-deps - $(call golang-dep-vendor) \ No newline at end of file + $(call golang-dep-vendor) diff --git a/batchconsumer/writer.go b/batchconsumer/writer.go index d750497..cf5eab4 100644 --- a/batchconsumer/writer.go +++ b/batchconsumer/writer.go @@ -58,7 +58,7 @@ func (b *batchedWriter) Initialize(shardID string, checkpointer kcl.Checkpointer func (b *batchedWriter) splitMessageIfNecessary(record []byte) ([][]byte, error) { // We handle two types of records: // - records emitted from CWLogs Subscription - // - records emiited from KPL + // - records emitted from KPL if !splitter.IsGzipped(record) { // Process a single message, from KPL return [][]byte{record}, nil diff --git a/golang.mk b/golang.mk index 8b0bb8b..7d8ba4c 100644 --- a/golang.mk +++ b/golang.mk @@ -1,7 +1,7 @@ # This is the default Clever Golang Makefile. # It is stored in the dev-handbook repo, github.com/Clever/dev-handbook # Please do not alter this file directly. -GOLANG_MK_VERSION := 0.2.0 +GOLANG_MK_VERSION := 0.3.2 SHELL := /bin/bash SYSTEM := $(shell uname -a | cut -d" " -f1 | tr '[:upper:]' '[:lower:]') @@ -18,7 +18,7 @@ define golang-version-check _ := $(if \ $(shell \ expr >/dev/null \ - `go version | cut -d" " -f3 | cut -c3- | cut -d. -f2` \ + `go version | cut -d" " -f3 | cut -c3- | cut -d. -f2 | sed -E 's/beta[0-9]+//'` \ \>= `echo $(1) | cut -d. -f2` \ \& \ `go version | cut -d" " -f3 | cut -c3- | cut -d. -f1` \ @@ -52,9 +52,16 @@ golang-dep-vendor-deps: bin/dep # golang-godep-vendor is a target for saving dependencies with the dep tool # to the vendor/ directory. All nested vendor/ directories are deleted via # the prune command. +# In CI, -vendor-only is used to avoid updating the lock file. +ifndef CI define golang-dep-vendor -bin/dep ensure +bin/dep ensure -v endef +else +define golang-dep-vendor +bin/dep ensure -v -vendor-only +endef +endif # Golint is a tool for linting Golang code for common errors. GOLINT := $(GOPATH)/bin/golint @@ -147,6 +154,19 @@ $(call golang-vet,$(1)) $(call golang-test-strict,$(1)) endef +# golang-build: builds a golang binary. ensures CGO build is done during CI. This is needed to make a binary that works with a Docker alpine image. +# arg1: pkg path +# arg2: executable name +define golang-build +@echo "BUILDING..." +@if [ -z "$$CI" ]; then \ + go build -o bin/$(2) $(1); \ +else \ + echo "-> Building CGO binary"; \ + CGO_ENABLED=0 go build -installsuffix cgo -o bin/$(2) $(1); \ +fi; +endef + # golang-update-makefile downloads latest version of golang.mk golang-update-makefile: @wget https://raw.githubusercontent.com/Clever/dev-handbook/master/make/golang.mk -O /tmp/golang.mk 2>/dev/null diff --git a/splitter/splitter.go b/splitter/splitter.go index 88f4a12..0a513ef 100644 --- a/splitter/splitter.go +++ b/splitter/splitter.go @@ -108,8 +108,9 @@ var awsBatchTaskRegex = regexp.MustCompile(awsBatchTaskMeta) var awsLambdaLogGroupRegex = regexp.MustCompile(`^/aws/lambda/([a-z0-9-]+)--([a-z0-9-]+)$`) var awsLambdaRequestIDRegex = regexp.MustCompile(`[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}`) -// arn cruft to satisfy parsing later on: https://github.com/Clever/amazon-kinesis-client-go/blob/94aacdf8339bd2cc8400d3bcb323dc1bce2c8422/decode/decode.go#L421-L425 +// arn and task cruft to satisfy parsing later on: https://github.com/Clever/amazon-kinesis-client-go/blob/94aacdf8339bd2cc8400d3bcb323dc1bce2c8422/decode/decode.go#L421-L425 const arnCruft = `/arn%3Aaws%3Aecs%3Aus-east-1%3A999988887777%3Atask%2F` +const taskCruft = `12345678-1234-1234-1234-555566667777` type RSysLogMessage struct { Timestamp time.Time @@ -180,8 +181,8 @@ func splitDefault(b LogEventBatch) []RSysLogMessage { for _, event := range b.LogEvents { out = append(out, RSysLogMessage{ Timestamp: event.Timestamp.Time(), - Hostname: "unknown", - ProgramName: "unknown", + Hostname: b.LogStream, + ProgramName: b.LogGroup + "--" + b.LogStream + arnCruft + taskCruft, PID: 1, Message: event.Message, }) diff --git a/splitter/splitter_test.go b/splitter/splitter_test.go index 4f9e512..81b2df2 100644 --- a/splitter/splitter_test.go +++ b/splitter/splitter_test.go @@ -153,3 +153,25 @@ func TestSplitLambda(t *testing.T) { } assert.Equal(t, expected, lines) } + +func TestSplitDefault(t *testing.T) { + input := LogEventBatch{ + MessageType: "DATA_MESSAGE", + Owner: "123456789012", + LogGroup: "vpn_flow_logs", + LogStream: "eni-43403819-all", + SubscriptionFilters: []string{"SomeSubscription"}, + LogEvents: []LogEvent{ + { + ID: "99999992379011144044923130086453437181614530551221780480", + Timestamp: NewUnixTimestampMillis(1498519943285), + Message: "2 589690932525 eni-43403819 10.0.0.233 172.217.6.46 64067 443 17 8 3969 1516891809 1516891868 ACCEPT OK", + }, + }, + } + lines := Split(input) + expected := [][]byte{ + []byte(`2017-06-26T23:32:23.285001+00:00 eni-43403819-all vpn_flow_logs--eni-43403819-all/arn%3Aaws%3Aecs%3Aus-east-1%3A999988887777%3Atask%2F12345678-1234-1234-1234-555566667777[1]: 2 589690932525 eni-43403819 10.0.0.233 172.217.6.46 64067 443 17 8 3969 1516891809 1516891868 ACCEPT OK`), + } + assert.Equal(t, expected, lines) +}