From df57dc6b3e5c4026ae9bb034fe1bf6d42f92cef3 Mon Sep 17 00:00:00 2001 From: Taylor Sutton Date: Wed, 19 Jul 2023 10:44:33 -0700 Subject: [PATCH 1/4] Update to Go 1.16 and fix broken go.mod mod declaration Plus `go mod tidy` --- .circleci/config.yml | 5 ++--- Makefile | 2 +- go.mod | 5 ++--- go.sum | 2 -- 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d1327b8..2cf292c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,10 +1,9 @@ version: 2 jobs: build: - working_directory: /go/src/github.com/Clever/amazon-kinesis-client-go + working_directory: ~/go/src/github.com/Clever/amazon-kinesis-client-go docker: - - image: circleci/golang:1.13-stretch - - image: circleci/mongo:3.2.20-jessie-ram + - image: cimg/go:1.16 environment: GOPRIVATE: github.com/Clever/* CIRCLE_ARTIFACTS: /tmp/circleci-artifacts diff --git a/Makefile b/Makefile index 5d0de9c..7d358d7 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ SHELL := /bin/bash PKG := github.com/Clever/amazon-kinesis-client-go PKGS := $(shell go list ./... | grep -v /vendor ) .PHONY: download_jars run build -$(eval $(call golang-version-check,1.13)) +$(eval $(call golang-version-check,1.16)) CONSUMER ?= consumer TMP_DIR := ./tmp-jars diff --git a/go.mod b/go.mod index faa1929..7258227 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,8 @@ -module github.com/amazon-kinesis-client-go +module github.com/Clever/amazon-kinesis-client-go -go 1.13 +go 1.16 require ( - github.com/Clever/amazon-kinesis-client-go v1.0.0 github.com/Clever/syslogparser v0.0.0-20170816194131-fb28ad3e4340 github.com/a8m/kinesis-producer v0.2.0 github.com/aws/aws-sdk-go v1.35.28 // indirect diff --git a/go.sum b/go.sum index 1703335..46c0c30 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,3 @@ -github.com/Clever/amazon-kinesis-client-go v1.0.0 h1:rG+hkHCSe+xMGk3asESg18tiEj8X6To8vj0TRa1pKvQ= -github.com/Clever/amazon-kinesis-client-go v1.0.0/go.mod h1:jLfi8QusUjdsYazmEH0DlSO4gE1WRzVE7cs4c/fOFdI= github.com/Clever/syslogparser v0.0.0-20170816194131-fb28ad3e4340 h1:wr8lTqPJZQpZkZgllQtrD96SecXkAu5MxzY3yJgsuCg= github.com/Clever/syslogparser v0.0.0-20170816194131-fb28ad3e4340/go.mod h1:e7Yy7RTiIMU9pZ+dcSviX3cpod8e0CEeTUPTgBbKlRE= github.com/a8m/kinesis-producer v0.2.0 h1:Bd5Oi4dczbTLPIZwVbm02464LIFgBqmViFj//b098xc= From e734611a60bc0a31bd10eebe20a24009b0ec212b Mon Sep 17 00:00:00 2001 From: Taylor Sutton Date: Wed, 19 Jul 2023 11:19:41 -0700 Subject: [PATCH 2/4] Fix gofmt. --- splitter/splitter.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/splitter/splitter.go b/splitter/splitter.go index abcf9ee..6f81ef8 100644 --- a/splitter/splitter.go +++ b/splitter/splitter.go @@ -4,9 +4,9 @@ // KCL applications would be most interested in `SplitMessageIfNecessary` which can handle zlibbed records as well as // CloudWatch bundles. KCL automatically unbundles KPL aggregates before passing the records to the consumer. // Non-KCL applications (such as Lambdas consuming KPL-produced aggregates) should either use -// - KPLDeaggregate if the consumer purely wants to unbundle KPL aggregates, but will handle the raw records themselves. -// - Deaggregate if the consumer wants to apply the same decompress and split logic as SplitMessageIfNecessary -// in addition to the KPL splitting. +// - KPLDeaggregate if the consumer purely wants to unbundle KPL aggregates, but will handle the raw records themselves. +// - Deaggregate if the consumer wants to apply the same decompress and split logic as SplitMessageIfNecessary +// in addition to the KPL splitting. package splitter import ( From de490c45abd068d10c44c5bb2e711f593fcade89 Mon Sep 17 00:00:00 2001 From: Taylor Sutton Date: Wed, 19 Jul 2023 11:19:50 -0700 Subject: [PATCH 3/4] Remove faulty error type checks. These type checks were not doing anything - which was surfaced by a `go vet` run on the more recent Go version. errors.As with the second argument of type *error will always return true. I played a little bit with trying to get a working typecheck in this test, but it didn't really work because of issues with the fork github.com/Clever/syslogparser. Since that parser didn't update self-references to point to itself instead of the upstream, our errors returned by this are already not really working correctly - sometimes they are Clever/syslogparser.ParseError and sometimes jeromer/syslogparser.ParseError. Fixing the underlying problem is out of scope for right now. --- decode/decode_test.go | 4 ---- golang.mk | 53 +++++++++++++++++++++++++++++++++++-------- 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/decode/decode_test.go b/decode/decode_test.go index 5a17b09..e9978c6 100644 --- a/decode/decode_test.go +++ b/decode/decode_test.go @@ -1,7 +1,6 @@ package decode import ( - "errors" "fmt" "sort" "testing" @@ -222,7 +221,6 @@ func TestSyslogDecoding(t *testing.T) { fields, err := FieldsFromSyslog(spec.Input) if spec.ExpectedError != nil { assert.Error(err) - assert.True(errors.As(err, &spec.ExpectedError)) return } assert.NoError(err) @@ -547,7 +545,6 @@ select sleep(2);`, fields, err := ParseAndEnhance(spec.Line, "deploy-env") if spec.ExpectedError != nil { assert.Error(err) - assert.True(errors.As(err, &spec.ExpectedError)) return } assert.NoError(err) @@ -557,7 +554,6 @@ select sleep(2);`, } func TestGetContainerMeta(t *testing.T) { - type containerMetaSpec struct { description string input map[string]interface{} diff --git a/golang.mk b/golang.mk index 18e211c..d22a169 100644 --- a/golang.mk +++ b/golang.mk @@ -1,7 +1,7 @@ # This is the default Clever Golang Makefile. # It is stored in the dev-handbook repo, github.com/Clever/dev-handbook # Please do not alter this file directly. -GOLANG_MK_VERSION := 1.0.0 +GOLANG_MK_VERSION := 1.1.0 SHELL := /bin/bash SYSTEM := $(shell uname -a | cut -d" " -f1 | tr '[:upper:]' '[:lower:]') @@ -11,7 +11,7 @@ SYSTEM := $(shell uname -a | cut -d" " -f1 | tr '[:upper:]' '[:lower:]') export TZ=UTC # go build flags for use across all commands which accept them -GO_BUILD_FLAGS := "-mod=vendor" +export GOFLAGS := -mod=vendor $(GOFLAGS) # if the gopath includes several directories, use only the first GOPATH=$(shell echo $$GOPATH | cut -d: -f1) @@ -39,7 +39,7 @@ endef # so we're defended against it breaking or changing in the future. FGT := $(GOPATH)/bin/fgt $(FGT): - go get github.com/GeertJohan/fgt@262f7b11eec07dc7b147c44641236f3212fee89d + go install -mod=readonly github.com/GeertJohan/fgt@262f7b11eec07dc7b147c44641236f3212fee89d golang-ensure-curl-installed: @command -v curl >/dev/null 2>&1 || { echo >&2 "curl not installed. Please install curl."; exit 1; } @@ -49,7 +49,7 @@ golang-ensure-curl-installed: # previously passing tests start failing without changing our code. GOLINT := $(GOPATH)/bin/golint $(GOLINT): - go get golang.org/x/lint/golint@738671d3881b9731cc63024d5d88cf28db875626 + go install -mod=readonly golang.org/x/lint/golint@738671d3881b9731cc63024d5d88cf28db875626 # golang-fmt-deps requires the FGT tool for checking output golang-fmt-deps: $(FGT) @@ -89,7 +89,7 @@ golang-test-deps: # arg1: pkg path define golang-test @echo "TESTING $(1)..." -@go test $(GO_BUILD_FLAGS) -v $(1) +@go test -v $(1) endef # golang-test-strict-deps is here for consistency @@ -99,7 +99,22 @@ golang-test-strict-deps: # arg1: pkg path define golang-test-strict @echo "TESTING $(1)..." -@go test -v $(GO_BUILD_FLAGS) -race $(1) +@go test -v -race $(1) +endef + +# golang-test-strict-cover-deps is here for consistency +golang-test-strict-cover-deps: + +# golang-test-strict-cover uses the Go toolchain to run all tests in the pkg with the race and cover flag. +# appends coverage results to coverage.txt +# arg1: pkg path +define golang-test-strict-cover +@echo "TESTING $(1)..." +@go test -v -race -cover -coverprofile=profile.tmp -covermode=atomic $(1) +@if [ -f profile.tmp ]; then \ + cat profile.tmp | tail -n +2 >> coverage.txt; \ + rm profile.tmp; \ +fi; endef # golang-vet-deps is here for consistency @@ -109,7 +124,7 @@ golang-vet-deps: # arg1: pkg path define golang-vet @echo "VETTING $(1)..." -@go vet $(GO_BUILD_FLAGS) $(1) +@go vet $(1) endef # golang-test-all-deps installs all dependencies needed for different test cases. @@ -137,19 +152,37 @@ $(call golang-vet,$(1)) $(call golang-test-strict,$(1)) endef +# golang-test-all-strict-cover-deps: installs all dependencies needed for different test cases. +golang-test-all-strict-cover-deps: golang-fmt-deps golang-lint-deps-strict golang-test-strict-cover-deps golang-vet-deps + +# golang-test-all-strict-cover calls fmt, lint, vet and test on the specified pkg with strict and cover +# requirements that no errors are thrown while linting. +# arg1: pkg path +define golang-test-all-strict-cover +$(call golang-fmt,$(1)) +$(call golang-lint-strict,$(1)) +$(call golang-vet,$(1)) +$(call golang-test-strict-cover,$(1)) +endef + # golang-build: builds a golang binary. ensures CGO build is done during CI. This is needed to make a binary that works with a Docker alpine image. # arg1: pkg path # arg2: executable name define golang-build -@echo "BUILDING..." +@echo "BUILDING $(2)..." @if [ -z "$$CI" ]; then \ - go build $(GO_BUILD_FLAGS) -o bin/$(2) $(1); \ + go build -o bin/$(2) $(1); \ else \ echo "-> Building CGO binary"; \ - CGO_ENABLED=0 go build $(GO_BUILD_FLAGS) -installsuffix cgo -o bin/$(2) $(1); \ + CGO_ENABLED=0 go build -installsuffix cgo -o bin/$(2) $(1); \ fi; endef +# golang-setup-coverage: set up the coverage file +define golang-setup-coverage: + @echo "mode: atomic" > coverage.txt +endef + # golang-update-makefile downloads latest version of golang.mk golang-update-makefile: @wget https://raw.githubusercontent.com/Clever/dev-handbook/master/make/golang-v1.mk -O /tmp/golang.mk 2>/dev/null From f8b8cb924eec1fdd72f42408c28615d8d563c022 Mon Sep 17 00:00:00 2001 From: Taylor Sutton Date: Wed, 19 Jul 2023 11:29:55 -0700 Subject: [PATCH 4/4] Bump version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 3eefcb9..9084fa2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.0 +1.1.0