From 6f0fbfe4c73070e7dbb2a10be589329124a8d5a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Grac=CC=A7as?= Date: Thu, 11 Nov 2021 23:08:59 +0100 Subject: [PATCH] improve ci (remove hypermake, add makefile and github pipeline) --- .github/workflows/vmware-go-kcl-v2-ci.yml | 42 +++++++++ .gitignore | 3 +- HyperMake | 91 ------------------- Makefile | 59 ++++++++++++ _support/scripts/ci.sh | 105 ++++++++++++++++++++++ _support/scripts/sonar-scan.sh | 63 +++++++++++++ go.sum | 2 - support/scripts/check.sh | 75 ---------------- support/scripts/ci.sh | 7 -- support/scripts/functions.sh | 59 ------------ support/scripts/test.sh | 5 -- support/toolchain/HyperMake | 28 ------ support/toolchain/docker/Dockerfile | 8 -- 13 files changed, 271 insertions(+), 276 deletions(-) create mode 100755 .github/workflows/vmware-go-kcl-v2-ci.yml delete mode 100644 HyperMake create mode 100644 Makefile create mode 100755 _support/scripts/ci.sh create mode 100644 _support/scripts/sonar-scan.sh delete mode 100755 support/scripts/check.sh delete mode 100755 support/scripts/ci.sh delete mode 100644 support/scripts/functions.sh delete mode 100755 support/scripts/test.sh delete mode 100644 support/toolchain/HyperMake delete mode 100644 support/toolchain/docker/Dockerfile diff --git a/.github/workflows/vmware-go-kcl-v2-ci.yml b/.github/workflows/vmware-go-kcl-v2-ci.yml new file mode 100755 index 0000000..137e2f0 --- /dev/null +++ b/.github/workflows/vmware-go-kcl-v2-ci.yml @@ -0,0 +1,42 @@ +name: vmware-go-kcl-v2 + +on: + push: + branches: [ master ] + paths-ignore: [ README.md ] + pull_request: + branches: [ master ] + paths-ignore: [ README.md ] + +jobs: + build: + name: Build + runs-on: ubuntu-latest + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + + - name: Build + shell: bash + run: | + make build + +# - name: Test +# shell: bash +# run: | +# make test + + - name: Format Check + shell: bash + run: | + make format-check + + - name: Lint + shell: bash + run: | + make lint + + - name: Scan + shell: bash + run: | + make scan \ No newline at end of file diff --git a/.gitignore b/.gitignore index a3b9045..e537833 100644 --- a/.gitignore +++ b/.gitignore @@ -21,4 +21,5 @@ *_mock_test.go filenames -.DS_Store \ No newline at end of file +.DS_Store +.scannerwork/ \ No newline at end of file diff --git a/HyperMake b/HyperMake deleted file mode 100644 index cffd91b..0000000 --- a/HyperMake +++ /dev/null @@ -1,91 +0,0 @@ ---- -format: hypermake.v0 - -name: cascade-kinesis-client -description: Kinesis Client in Go - -targets: - rebuild-toolchain: - description: build toolchain image - watches: - - support/toolchain/docker - build: support/toolchain/docker - - toolchain: - description: placeholder for additional toolchain dependencies - - deps: - description: download dependencies to local cache - after: - - toolchain - watches: - - go.mod - cmds: - - go mod download - - go mod vendor - - go mod tidy - - build: - description: build source code - after: - - 'build-*' - - test: - description: run unit tests - after: - - deps - - check - always: true - cmds: - - ./support/scripts/test.sh - - ci: - description: run CI tests - after: - - deps - cmds: - - ./support/scripts/ci.sh - - checkfmt: - description: check code format - after: - - toolchain - watches: - - support/scripts/check.sh - always: true - cmds: - - ./support/scripts/check.sh fmt - - lint: - description: run lint to check code - after: - - toolchain - watches: - - support/scripts/check.sh - always: true - cmds: - - ./support/scripts/check.sh lint - - scanast: - description: run Go AST security scan - after: - - toolchain - watches: - - '**/**/*.go' - - './support/scripts/check.sh' - cmds: - - ./support/scripts/check.sh scanast - - check: - description: run all code checks - after: - - checkfmt - - lint - - scanast - -settings: - default-targets: - - test - docker: - image: 'vmware/go-kcl-toolchain:0.1.4' - src-volume: /go/src/github.com/vmware/vmware-go-kcl diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..78026f1 --- /dev/null +++ b/Makefile @@ -0,0 +1,59 @@ +.PHONY: help +help: ## - Show this help message + @printf "\033[32m\xE2\x9c\x93 usage: make [target]\n\n\033[0m" + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + +.PHONY: up +up: ## - start docker compose + @ cd _support/docker && docker-compose -f docker-compose.yml up + +.PHONY: build-common +build-common: ## - execute build common tasks clean and mod tidy + @ go version + @ go clean + @ go mod download && go mod tidy + @ go mod verify + +.PHONY: build +build: build-common ## - build a debug binary to the current platform (windows, linux or darwin(mac)) + @ echo building + @ go build -v ./... + @ echo "done" + +.PHONY: format-check +format-check: ## - check files format using gofmt + @ ./_support/scripts/ci.sh fmtcheck + +.PHONY: format-check +format: ## - apply golang file format using gofmt + @ ./_support/scripts/ci.sh format + +.PHONY: test +test: build-common ## - execute go test command for unit and mocked tests + @ ./_support/scripts/ci.sh unittest + +.PHONY: integration-test +integration-test: ## - execute go test command for integration tests (aws credentials needed) + @ go test -v -cover -race ./test + +.PHONY: scan +scan: ## - execute static code analysis + @ ./_support/scripts/ci.sh scan + +.PHONY: lint +lint: ## - runs golangci-lint + @ ./_support/scripts/ci.sh lint + +.PHONY: ci-lint-docker +lint-docker: ## - runs golangci-lint with docker container + @ docker run --rm -v "$(shell pwd)":/app -w /app ${LINT_IMAGE} golangci-lint run ${LINT_FLAGS} + +.PHONY: sonar-scan +sonar-scan: ## - start sonar qube locally with docker (you will need docker installed in your machine) + @ # after start, setup a new project with the name sms-local and a new token sms-token, fill the token against the -Dsonar.login= parameter. + @ # login with user: admin pwd: vmware + @ $(SHELL) _support/scripts/sonar-scan.sh + +.PHONY: sonar-stop +sonar-stop: ## - stop sonar qube docker container + @ docker stop sonarqube diff --git a/_support/scripts/ci.sh b/_support/scripts/ci.sh new file mode 100755 index 0000000..e53d247 --- /dev/null +++ b/_support/scripts/ci.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash + +function local_go_pkgs() { + find './clientlibrary' -name '*.go' | \ + grep -Fv '/vendor/' | \ + grep -Fv '/go/' | \ + grep -Fv '/gen/' | \ + grep -Fv '/tmp/' | \ + grep -Fv '/run/' | \ + grep -Fv '/tests/' | \ + sed -r 's|(.+)/[^/]+\.go$|\1|g' | \ + sort -u +} + +function checkfmt() { + local files="" + files="$(find . -type f -iname "*.go" -exec gofmt -l {} \;)" + + if [ -n "$files" ]; then + echo "You need to run \"gofmt -w ./\" to fix your formatting." + echo "$files" >&2 + return 1 + fi +} + +function go_format() { + echo "go formatting..." + gofmt -w ./ + echo "done" +} + +function lint() { + # golangci-lint run --enable-all -D forbidigo -D gochecknoglobals -D gofumpt -D gofmt -D nlreturn + golangci-lint run \ + --skip-files=_mock.go \ + --disable=golint \ + --skip-dirs=test \ + --fast \ + --timeout=600s \ + --verbose \ + "$(local_go_pkgs)" +} + +function test() { + go list ./... | grep -v /test | \ + xargs -L 1 -I% bash -c 'echo -e "\n**************** Package: % ****************" && go test % -v -cover -race ./...' +} + +function scanast() { + gosec version + gosec ./... > security.log 2>&1 + + local issues="" + issues=$(grep -c 'Severity: MEDIUM' security.log | grep -v deaggregator | grep -c _) + if [ -n "$issues" ] && [ "$issues" -gt 0 ]; then + echo "" + echo "Medium Severity Issues:" + grep -e "Severity: MEDIUM" -A 1 security.log + echo "$issues" "medium severity issues found." + fi + + local issues="" + local issues_count="" + issues="$(grep -E 'Severity: HIGH' security.log | grep -v vendor)" + issues_count="$(grep -E 'Severity: HIGH' security.log | grep -v vendor | grep -c _)" + if [ -n "$issues_count" ] && [ "$issues_count" -gt 0 ]; then + echo "" + echo "High Severity Issues:" + grep -E "Severity: HIGH" -A 1 security.log + echo "$issues_count" "high severity issues found." + echo "$issues" + echo "You need to resolve the high severity issues at the least." + exit 1 + fi + + local issues="" + local issues_count="" + issues="$(grep -E 'Errors unhandled' security.log | grep -v vendor | grep -v /src/go/src)" + issues_count="$(grep -E 'Errors unhandled' security.log | grep -v vendor | grep -v /src/go/src | grep -c _)" + if [ -n "$issues_count" ] && [ "$issues_count" -gt 0 ]; then + echo "" + echo "Unhandled errors:" + grep -E "Errors unhandled" security.log + echo "$issues_count" "unhandled errors, please indicate with the right comment that this case is ok, or handle the error." + echo "$issues" + echo "You need to resolve the all unhandled errors." + exit 1 + fi + + rm -f security.log +} + +function usage() { + echo "check.sh fmt|lint" >&2 + exit 2 +} + +case "$1" in + fmtcheck) checkfmt ;; + format) go_format ;; + lint) lint ;; + unittest) test ;; + scan) scanast ;; + *) usage ;; +esac diff --git a/_support/scripts/sonar-scan.sh b/_support/scripts/sonar-scan.sh new file mode 100644 index 0000000..4fcb34a --- /dev/null +++ b/_support/scripts/sonar-scan.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +######################## +# requirements: # +# 0. docker # +# 1. wget # +# 2. curl # +# 3. jq # +# 4. sonar-scanner # +######################## + +set -e + +projectKey="vmware-go-kcl-v2" +user_tokenName="local_token" +username="admin" +user_password="admin" +new_password="vmware" +url="http://localhost" +port="9000" + +if [[ "$( docker container inspect -f '{{.State.Running}}' sonarqube )" == "true" ]]; +then + docker ps +else + docker run --rm -d --name sonarqube -e SONAR_ES_BOOTSTRAP_CHECKS_DISABLE=true -p 9000:9000 sonarqube +fi + +echo "waiting for sonarqube starts..." +wget -q -O - "$@" http://localhost:9000 | awk '/STARTING/{ print $0 }' | xargs + +STATUS="$(wget -q -O - "$@" http://localhost:9000 | awk '/UP/{ print $0 }')" +while [ -z "$STATUS" ] +do + sleep 2 + STATUS="$(wget -q -O - "$@" http://localhost:9000 | awk '/UP/{ print $0 }')" + printf "." +done + +printf '\n %s' "${STATUS}" | xargs +echo "" + +# change the default password to avoid create a new one when login for the very first time +curl -u ${username}:${user_password} -X POST "${url}:${port}/api/users/change_password?login=${username}&previousPassword=${user_password}&password=${new_password}" + +# search the specific user tokens for SonarQube +hasToken=$(curl --silent -u ${username}:${new_password} -X GET "${url}:${port}/api/user_tokens/search") +if [[ -n "${hasToken}" ]]; then + # Revoke the user token for SonarQube + curl -X POST -H "Content-Type: application/x-www-form-urlencoded" -d "name=${user_tokenName}" -u ${username}:${new_password} "${url}:${port}"/api/user_tokens/revoke +fi + +# generate new token +token=$(curl --silent -X POST -H "Content-Type: application/x-www-form-urlencoded" -d "name=${user_tokenName}" -u ${username}:${new_password} "${url}:${port}"/api/user_tokens/generate | jq '.token' | xargs) + +# scan and push the results to localhost docker container +sonar-scanner -Dsonar.projectKey="${projectKey}" \ + -Dsonar.projectName="${projectKey}" \ + -Dsonar.sources=. \ + -Dsonar.exclusions="internal/records/**, test/**" \ + -Dsonar.host.url="${url}:${port}" \ + -Dsonar.login="${token}" + diff --git a/go.sum b/go.sum index 9a7eb1b..6030020 100644 --- a/go.sum +++ b/go.sum @@ -383,8 +383,6 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211106132015-ebca88c72f68 h1:Ywe/f3fNleF8I6F6qv3MeFoSZ6CTf2zBMMa/7qVML8M= -golang.org/x/sys v0.0.0-20211106132015-ebca88c72f68/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211107104306-e0b2ad06fe42 h1:G2DDmludOQZoWbpCr7OKDxnl478ZBGMcOhrv+ooX/Q4= golang.org/x/sys v0.0.0-20211107104306-e0b2ad06fe42/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/support/scripts/check.sh b/support/scripts/check.sh deleted file mode 100755 index eb87f6b..0000000 --- a/support/scripts/check.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env bash - -. support/scripts/functions.sh - -checkfmt() { - local files="$(gofmt -l $(local_go_pkgs))" - if [ -n "$files" ]; then - echo "You need to run \"gofmt -w ./\" to fix your formating." - echo "$files" >&2 - return 1 - fi -} - -lint() { - golangci-lint run \ - --skip-files=_mock.go \ - --disable=golint \ - --skip-dirs=test \ - --fast \ - --timeout=600s \ - --verbose \ - $(local_go_pkgs) -} - -scanast() { - set +e - gosec version - gosec ./... > security.log 2>&1 - set -e - - local issues="$(grep -E 'Severity: MEDIUM' security.log | wc -l)" - if [ -n $issues ] && [ $issues -gt 0 ]; then - echo "" - echo "Medium Severity Issues:" - grep -E "Severity: MEDIUM" -A 1 security.log - echo $issues "medium severity issues found." - fi - - local issues="$(grep -E 'Severity: HIGH' security.log | grep -v vendor)" - local issues_count="$(grep -E 'Severity: HIGH' security.log | grep -v vendor | wc -l)" - if [ -n $issues_count ] && [ $issues_count -gt 0 ]; then - echo "" - echo "High Severity Issues:" - grep -E "Severity: HIGH" -A 1 security.log - echo $issues_count "high severity issues found." - echo $issues - echo "You need to resolve the high severity issues at the least." - exit 1 - fi - - local issues="$(grep -E 'Errors unhandled' security.log | grep -v vendor | grep -v /src/go/src)" - local issues_count="$(grep -E 'Errors unhandled' security.log | grep -v vendor | grep -v /src/go/src | wc -l)" - if [ -n $issues_count ] && [ $issues_count -gt 0 ]; then - echo "" - echo "Unhandled errors:" - grep -E "Errors unhandled" security.log - echo $issues_count "unhandled errors, please indicate with the right comment that this case is ok, or handle the error." - echo $issues - echo "You need to resolve the all unhandled errors." - exit 1 - fi - rm security.log -} - -usage() { - echo "check.sh fmt|lint" >&2 - exit 2 -} - -case "$1" in - fmt) checkfmt ;; - lint) lint ;; - scanast) scanast;; - *) usage ;; -esac diff --git a/support/scripts/ci.sh b/support/scripts/ci.sh deleted file mode 100755 index 420bde0..0000000 --- a/support/scripts/ci.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -# Run only the integration tests -# go test -race ./test -echo "Warning: Cannot find a good way to inject AWS credential to hmake container" -echo "Don't use hmake ci. Use the following command directly" -echo "go test -race ./test" diff --git a/support/scripts/functions.sh b/support/scripts/functions.sh deleted file mode 100644 index 489de81..0000000 --- a/support/scripts/functions.sh +++ /dev/null @@ -1,59 +0,0 @@ -set -ex - -# PROJ_ROOT specifies the project root -export PROJ_ROOT="$HMAKE_PROJECT_DIR" - -# Add /go in GOPATH because that's the original GOPATH in toolchain -export GOPATH=/go:$PROJ_ROOT - -local_go_pkgs() { - find './clientlibrary/' -name '*.go' | \ - grep -Fv '/vendor/' | \ - grep -Fv '/go/' | \ - grep -Fv '/gen/' | \ - grep -Fv '/tmp/' | \ - grep -Fv '/run/' | \ - grep -Fv '/tests/' | \ - sed -r 's|(.+)/[^/]+\.go$|\1|g' | \ - sort -u -} - -version_suffix() { - local suffix=$(git log -1 --format=%h 2>/dev/null || true) - if [ -n "$suffix" ]; then - test -z "$(git status --porcelain 2>/dev/null || true)" || suffix="${suffix}+" - echo -n "-g${suffix}" - else - echo -n -dev - fi -} - -git_commit_hash() { - echo $(git rev-parse --short HEAD) -} - -# Due to Go plugin genhash algorithm simply takes full source path -# from archive, it generates different plugin hash if source path of -# shared pkg is different, and causes load failure. -# as a workaround, lookup shared pkg and place it to fixed path -FIX_GOPATH=/tmp/go - -fix_go_pkg() { - local pkg="$1" base - for p in ${GOPATH//:/ }; do - if [ -d "$p/src/$pkg" ]; then - base="$p" - break - fi - done - - if [ -z "$base" ]; then - echo "Package $pkg not found in GOPATH: $GOPATH" >&2 - return 1 - fi - - local fix_pkg_path="$FIX_GOPATH/src/$pkg" - rm -f "$fix_pkg_path" - mkdir -p "$(dirname $fix_pkg_path)" - ln -s "$base/src/$pkg" "$fix_pkg_path" -} diff --git a/support/scripts/test.sh b/support/scripts/test.sh deleted file mode 100755 index ee8226e..0000000 --- a/support/scripts/test.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -. support/scripts/functions.sh - -# Run only the unit tests and not integration tests -go test -cover -race $(local_go_pkgs) diff --git a/support/toolchain/HyperMake b/support/toolchain/HyperMake deleted file mode 100644 index c294e5a..0000000 --- a/support/toolchain/HyperMake +++ /dev/null @@ -1,28 +0,0 @@ ---- -format: hypermake.v0 - -name: go-kcl -description: VMWare Go-KCL Amazon Kinesis Client Library in Go - -targets: - rebuild-toolchain: - description: build toolchain image - watches: - - docker - build: docker - cache: false - tags: - - vmware/go-kcl-toolchain:latest - - push-toolchain: - description: push toolchain image - after: - - rebuild-toolchain - push: - - vmware/go-kcl-toolchain:latest - -settings: - default-targets: - - rebuild-toolchain - docker: - image: 'vmware/go-kcl-toolchain:0.1.4' diff --git a/support/toolchain/docker/Dockerfile b/support/toolchain/docker/Dockerfile deleted file mode 100644 index 47a5d12..0000000 --- a/support/toolchain/docker/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM golang:1.17 -ENV PATH /go/bin:/src/bin:/root/go/bin:/usr/local/go/bin:$PATH -ENV GOPATH /go:/src -RUN go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.42.1 && \ - go install golang.org/x/tools/cmd/...@latest && \ - go install github.com/go-delve/delve/cmd/dlv@latest && \ - curl -sfL https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s v2.8.1 && \ - chmod -R a+rw /go