improve ci (remove hypermake, add makefile and github pipeline)

This commit is contained in:
Fabiano Graças 2021-11-11 23:08:59 +01:00 committed by Tao Jiang
parent 7538535bff
commit 6f0fbfe4c7
13 changed files with 271 additions and 276 deletions

42
.github/workflows/vmware-go-kcl-v2-ci.yml vendored Executable file
View file

@ -0,0 +1,42 @@
name: vmware-go-kcl-v2
on:
push:
branches: [ master ]
paths-ignore: [ README.md ]
pull_request:
branches: [ master ]
paths-ignore: [ README.md ]
jobs:
build:
name: Build
runs-on: ubuntu-latest
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v2
- name: Build
shell: bash
run: |
make build
# - name: Test
# shell: bash
# run: |
# make test
- name: Format Check
shell: bash
run: |
make format-check
- name: Lint
shell: bash
run: |
make lint
- name: Scan
shell: bash
run: |
make scan

3
.gitignore vendored
View file

@ -21,4 +21,5 @@
*_mock_test.go
filenames
.DS_Store
.DS_Store
.scannerwork/

View file

@ -1,91 +0,0 @@
---
format: hypermake.v0
name: cascade-kinesis-client
description: Kinesis Client in Go
targets:
rebuild-toolchain:
description: build toolchain image
watches:
- support/toolchain/docker
build: support/toolchain/docker
toolchain:
description: placeholder for additional toolchain dependencies
deps:
description: download dependencies to local cache
after:
- toolchain
watches:
- go.mod
cmds:
- go mod download
- go mod vendor
- go mod tidy
build:
description: build source code
after:
- 'build-*'
test:
description: run unit tests
after:
- deps
- check
always: true
cmds:
- ./support/scripts/test.sh
ci:
description: run CI tests
after:
- deps
cmds:
- ./support/scripts/ci.sh
checkfmt:
description: check code format
after:
- toolchain
watches:
- support/scripts/check.sh
always: true
cmds:
- ./support/scripts/check.sh fmt
lint:
description: run lint to check code
after:
- toolchain
watches:
- support/scripts/check.sh
always: true
cmds:
- ./support/scripts/check.sh lint
scanast:
description: run Go AST security scan
after:
- toolchain
watches:
- '**/**/*.go'
- './support/scripts/check.sh'
cmds:
- ./support/scripts/check.sh scanast
check:
description: run all code checks
after:
- checkfmt
- lint
- scanast
settings:
default-targets:
- test
docker:
image: 'vmware/go-kcl-toolchain:0.1.4'
src-volume: /go/src/github.com/vmware/vmware-go-kcl

59
Makefile Normal file
View file

@ -0,0 +1,59 @@
.PHONY: help
help: ## - Show this help message
@printf "\033[32m\xE2\x9c\x93 usage: make [target]\n\n\033[0m"
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
.PHONY: up
up: ## - start docker compose
@ cd _support/docker && docker-compose -f docker-compose.yml up
.PHONY: build-common
build-common: ## - execute build common tasks clean and mod tidy
@ go version
@ go clean
@ go mod download && go mod tidy
@ go mod verify
.PHONY: build
build: build-common ## - build a debug binary to the current platform (windows, linux or darwin(mac))
@ echo building
@ go build -v ./...
@ echo "done"
.PHONY: format-check
format-check: ## - check files format using gofmt
@ ./_support/scripts/ci.sh fmtcheck
.PHONY: format-check
format: ## - apply golang file format using gofmt
@ ./_support/scripts/ci.sh format
.PHONY: test
test: build-common ## - execute go test command for unit and mocked tests
@ ./_support/scripts/ci.sh unittest
.PHONY: integration-test
integration-test: ## - execute go test command for integration tests (aws credentials needed)
@ go test -v -cover -race ./test
.PHONY: scan
scan: ## - execute static code analysis
@ ./_support/scripts/ci.sh scan
.PHONY: lint
lint: ## - runs golangci-lint
@ ./_support/scripts/ci.sh lint
.PHONY: ci-lint-docker
lint-docker: ## - runs golangci-lint with docker container
@ docker run --rm -v "$(shell pwd)":/app -w /app ${LINT_IMAGE} golangci-lint run ${LINT_FLAGS}
.PHONY: sonar-scan
sonar-scan: ## - start sonar qube locally with docker (you will need docker installed in your machine)
@ # after start, setup a new project with the name sms-local and a new token sms-token, fill the token against the -Dsonar.login= parameter.
@ # login with user: admin pwd: vmware
@ $(SHELL) _support/scripts/sonar-scan.sh
.PHONY: sonar-stop
sonar-stop: ## - stop sonar qube docker container
@ docker stop sonarqube

105
_support/scripts/ci.sh Executable file
View file

@ -0,0 +1,105 @@
#!/usr/bin/env bash
function local_go_pkgs() {
find './clientlibrary' -name '*.go' | \
grep -Fv '/vendor/' | \
grep -Fv '/go/' | \
grep -Fv '/gen/' | \
grep -Fv '/tmp/' | \
grep -Fv '/run/' | \
grep -Fv '/tests/' | \
sed -r 's|(.+)/[^/]+\.go$|\1|g' | \
sort -u
}
function checkfmt() {
local files=""
files="$(find . -type f -iname "*.go" -exec gofmt -l {} \;)"
if [ -n "$files" ]; then
echo "You need to run \"gofmt -w ./\" to fix your formatting."
echo "$files" >&2
return 1
fi
}
function go_format() {
echo "go formatting..."
gofmt -w ./
echo "done"
}
function lint() {
# golangci-lint run --enable-all -D forbidigo -D gochecknoglobals -D gofumpt -D gofmt -D nlreturn
golangci-lint run \
--skip-files=_mock.go \
--disable=golint \
--skip-dirs=test \
--fast \
--timeout=600s \
--verbose \
"$(local_go_pkgs)"
}
function test() {
go list ./... | grep -v /test | \
xargs -L 1 -I% bash -c 'echo -e "\n**************** Package: % ****************" && go test % -v -cover -race ./...'
}
function scanast() {
gosec version
gosec ./... > security.log 2>&1
local issues=""
issues=$(grep -c 'Severity: MEDIUM' security.log | grep -v deaggregator | grep -c _)
if [ -n "$issues" ] && [ "$issues" -gt 0 ]; then
echo ""
echo "Medium Severity Issues:"
grep -e "Severity: MEDIUM" -A 1 security.log
echo "$issues" "medium severity issues found."
fi
local issues=""
local issues_count=""
issues="$(grep -E 'Severity: HIGH' security.log | grep -v vendor)"
issues_count="$(grep -E 'Severity: HIGH' security.log | grep -v vendor | grep -c _)"
if [ -n "$issues_count" ] && [ "$issues_count" -gt 0 ]; then
echo ""
echo "High Severity Issues:"
grep -E "Severity: HIGH" -A 1 security.log
echo "$issues_count" "high severity issues found."
echo "$issues"
echo "You need to resolve the high severity issues at the least."
exit 1
fi
local issues=""
local issues_count=""
issues="$(grep -E 'Errors unhandled' security.log | grep -v vendor | grep -v /src/go/src)"
issues_count="$(grep -E 'Errors unhandled' security.log | grep -v vendor | grep -v /src/go/src | grep -c _)"
if [ -n "$issues_count" ] && [ "$issues_count" -gt 0 ]; then
echo ""
echo "Unhandled errors:"
grep -E "Errors unhandled" security.log
echo "$issues_count" "unhandled errors, please indicate with the right comment that this case is ok, or handle the error."
echo "$issues"
echo "You need to resolve the all unhandled errors."
exit 1
fi
rm -f security.log
}
function usage() {
echo "check.sh fmt|lint" >&2
exit 2
}
case "$1" in
fmtcheck) checkfmt ;;
format) go_format ;;
lint) lint ;;
unittest) test ;;
scan) scanast ;;
*) usage ;;
esac

View file

@ -0,0 +1,63 @@
#!/usr/bin/env bash
########################
# requirements: #
# 0. docker #
# 1. wget #
# 2. curl #
# 3. jq #
# 4. sonar-scanner #
########################
set -e
projectKey="vmware-go-kcl-v2"
user_tokenName="local_token"
username="admin"
user_password="admin"
new_password="vmware"
url="http://localhost"
port="9000"
if [[ "$( docker container inspect -f '{{.State.Running}}' sonarqube )" == "true" ]];
then
docker ps
else
docker run --rm -d --name sonarqube -e SONAR_ES_BOOTSTRAP_CHECKS_DISABLE=true -p 9000:9000 sonarqube
fi
echo "waiting for sonarqube starts..."
wget -q -O - "$@" http://localhost:9000 | awk '/STARTING/{ print $0 }' | xargs
STATUS="$(wget -q -O - "$@" http://localhost:9000 | awk '/UP/{ print $0 }')"
while [ -z "$STATUS" ]
do
sleep 2
STATUS="$(wget -q -O - "$@" http://localhost:9000 | awk '/UP/{ print $0 }')"
printf "."
done
printf '\n %s' "${STATUS}" | xargs
echo ""
# change the default password to avoid create a new one when login for the very first time
curl -u ${username}:${user_password} -X POST "${url}:${port}/api/users/change_password?login=${username}&previousPassword=${user_password}&password=${new_password}"
# search the specific user tokens for SonarQube
hasToken=$(curl --silent -u ${username}:${new_password} -X GET "${url}:${port}/api/user_tokens/search")
if [[ -n "${hasToken}" ]]; then
# Revoke the user token for SonarQube
curl -X POST -H "Content-Type: application/x-www-form-urlencoded" -d "name=${user_tokenName}" -u ${username}:${new_password} "${url}:${port}"/api/user_tokens/revoke
fi
# generate new token
token=$(curl --silent -X POST -H "Content-Type: application/x-www-form-urlencoded" -d "name=${user_tokenName}" -u ${username}:${new_password} "${url}:${port}"/api/user_tokens/generate | jq '.token' | xargs)
# scan and push the results to localhost docker container
sonar-scanner -Dsonar.projectKey="${projectKey}" \
-Dsonar.projectName="${projectKey}" \
-Dsonar.sources=. \
-Dsonar.exclusions="internal/records/**, test/**" \
-Dsonar.host.url="${url}:${port}" \
-Dsonar.login="${token}"

2
go.sum
View file

@ -383,8 +383,6 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211106132015-ebca88c72f68 h1:Ywe/f3fNleF8I6F6qv3MeFoSZ6CTf2zBMMa/7qVML8M=
golang.org/x/sys v0.0.0-20211106132015-ebca88c72f68/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211107104306-e0b2ad06fe42 h1:G2DDmludOQZoWbpCr7OKDxnl478ZBGMcOhrv+ooX/Q4=
golang.org/x/sys v0.0.0-20211107104306-e0b2ad06fe42/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

View file

@ -1,75 +0,0 @@
#!/usr/bin/env bash
. support/scripts/functions.sh
checkfmt() {
local files="$(gofmt -l $(local_go_pkgs))"
if [ -n "$files" ]; then
echo "You need to run \"gofmt -w ./\" to fix your formating."
echo "$files" >&2
return 1
fi
}
lint() {
golangci-lint run \
--skip-files=_mock.go \
--disable=golint \
--skip-dirs=test \
--fast \
--timeout=600s \
--verbose \
$(local_go_pkgs)
}
scanast() {
set +e
gosec version
gosec ./... > security.log 2>&1
set -e
local issues="$(grep -E 'Severity: MEDIUM' security.log | wc -l)"
if [ -n $issues ] && [ $issues -gt 0 ]; then
echo ""
echo "Medium Severity Issues:"
grep -E "Severity: MEDIUM" -A 1 security.log
echo $issues "medium severity issues found."
fi
local issues="$(grep -E 'Severity: HIGH' security.log | grep -v vendor)"
local issues_count="$(grep -E 'Severity: HIGH' security.log | grep -v vendor | wc -l)"
if [ -n $issues_count ] && [ $issues_count -gt 0 ]; then
echo ""
echo "High Severity Issues:"
grep -E "Severity: HIGH" -A 1 security.log
echo $issues_count "high severity issues found."
echo $issues
echo "You need to resolve the high severity issues at the least."
exit 1
fi
local issues="$(grep -E 'Errors unhandled' security.log | grep -v vendor | grep -v /src/go/src)"
local issues_count="$(grep -E 'Errors unhandled' security.log | grep -v vendor | grep -v /src/go/src | wc -l)"
if [ -n $issues_count ] && [ $issues_count -gt 0 ]; then
echo ""
echo "Unhandled errors:"
grep -E "Errors unhandled" security.log
echo $issues_count "unhandled errors, please indicate with the right comment that this case is ok, or handle the error."
echo $issues
echo "You need to resolve the all unhandled errors."
exit 1
fi
rm security.log
}
usage() {
echo "check.sh fmt|lint" >&2
exit 2
}
case "$1" in
fmt) checkfmt ;;
lint) lint ;;
scanast) scanast;;
*) usage ;;
esac

View file

@ -1,7 +0,0 @@
#!/bin/bash
# Run only the integration tests
# go test -race ./test
echo "Warning: Cannot find a good way to inject AWS credential to hmake container"
echo "Don't use hmake ci. Use the following command directly"
echo "go test -race ./test"

View file

@ -1,59 +0,0 @@
set -ex
# PROJ_ROOT specifies the project root
export PROJ_ROOT="$HMAKE_PROJECT_DIR"
# Add /go in GOPATH because that's the original GOPATH in toolchain
export GOPATH=/go:$PROJ_ROOT
local_go_pkgs() {
find './clientlibrary/' -name '*.go' | \
grep -Fv '/vendor/' | \
grep -Fv '/go/' | \
grep -Fv '/gen/' | \
grep -Fv '/tmp/' | \
grep -Fv '/run/' | \
grep -Fv '/tests/' | \
sed -r 's|(.+)/[^/]+\.go$|\1|g' | \
sort -u
}
version_suffix() {
local suffix=$(git log -1 --format=%h 2>/dev/null || true)
if [ -n "$suffix" ]; then
test -z "$(git status --porcelain 2>/dev/null || true)" || suffix="${suffix}+"
echo -n "-g${suffix}"
else
echo -n -dev
fi
}
git_commit_hash() {
echo $(git rev-parse --short HEAD)
}
# Due to Go plugin genhash algorithm simply takes full source path
# from archive, it generates different plugin hash if source path of
# shared pkg is different, and causes load failure.
# as a workaround, lookup shared pkg and place it to fixed path
FIX_GOPATH=/tmp/go
fix_go_pkg() {
local pkg="$1" base
for p in ${GOPATH//:/ }; do
if [ -d "$p/src/$pkg" ]; then
base="$p"
break
fi
done
if [ -z "$base" ]; then
echo "Package $pkg not found in GOPATH: $GOPATH" >&2
return 1
fi
local fix_pkg_path="$FIX_GOPATH/src/$pkg"
rm -f "$fix_pkg_path"
mkdir -p "$(dirname $fix_pkg_path)"
ln -s "$base/src/$pkg" "$fix_pkg_path"
}

View file

@ -1,5 +0,0 @@
#!/bin/bash
. support/scripts/functions.sh
# Run only the unit tests and not integration tests
go test -cover -race $(local_go_pkgs)

View file

@ -1,28 +0,0 @@
---
format: hypermake.v0
name: go-kcl
description: VMWare Go-KCL Amazon Kinesis Client Library in Go
targets:
rebuild-toolchain:
description: build toolchain image
watches:
- docker
build: docker
cache: false
tags:
- vmware/go-kcl-toolchain:latest
push-toolchain:
description: push toolchain image
after:
- rebuild-toolchain
push:
- vmware/go-kcl-toolchain:latest
settings:
default-targets:
- rebuild-toolchain
docker:
image: 'vmware/go-kcl-toolchain:0.1.4'

View file

@ -1,8 +0,0 @@
FROM golang:1.17
ENV PATH /go/bin:/src/bin:/root/go/bin:/usr/local/go/bin:$PATH
ENV GOPATH /go:/src
RUN go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.42.1 && \
go install golang.org/x/tools/cmd/...@latest && \
go install github.com/go-delve/delve/cmd/dlv@latest && \
curl -sfL https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s v2.8.1 && \
chmod -R a+rw /go