improve ci (remove hypermake, add makefile and github pipeline)

.github/workflows/vmware-go-kcl-v2-ci.yml

@@ -0,0 +1,42 @@
+name: vmware-go-kcl-v2
+
+on:
+  push:
+    branches: [ master ]
+    paths-ignore: [ README.md ]
+  pull_request:
+    branches: [ master ]
+    paths-ignore: [ README.md ]
+    
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v2
+
+    - name: Build
+      shell: bash
+      run: |
+        make build
+
+#    - name: Test
+#      shell: bash
+#      run: |
+#        make test
+
+    - name: Format Check
+      shell: bash
+      run: |
+        make format-check
+
+    - name: Lint
+      shell: bash
+      run: |
+        make lint
+
+    - name: Scan
+      shell: bash
+      run: |
+        make scan

.gitignore

@@ -21,4 +21,5 @@
 *_mock_test.go
 filenames
 
-.DS_Store
+.DS_Store
+.scannerwork/

HyperMake

@@ -1,91 +0,0 @@
----
-format: hypermake.v0
-
-name: cascade-kinesis-client
-description: Kinesis Client in Go
-
-targets:
-  rebuild-toolchain:
-    description: build toolchain image
-    watches:
-      - support/toolchain/docker
-    build: support/toolchain/docker
-
-  toolchain:
-    description: placeholder for additional toolchain dependencies
-
-  deps:
-    description: download dependencies to local cache
-    after:
-      - toolchain
-    watches:
-      - go.mod
-    cmds:
-      - go mod download
-      - go mod vendor
-      - go mod tidy
-
-  build:
-    description: build source code
-    after:
-      - 'build-*'
-
-  test:
-    description: run unit tests
-    after:
-      - deps
-      - check
-    always: true
-    cmds:
-      - ./support/scripts/test.sh
-
-  ci:
-    description: run CI tests
-    after:
-      - deps
-    cmds:
-      - ./support/scripts/ci.sh
-
-  checkfmt:
-    description: check code format
-    after:
-      - toolchain
-    watches:
-      - support/scripts/check.sh
-    always: true
-    cmds:
-      - ./support/scripts/check.sh fmt
-
-  lint:
-    description: run lint to check code
-    after:
-      - toolchain
-    watches:
-      - support/scripts/check.sh
-    always: true
-    cmds:
-      - ./support/scripts/check.sh lint
-
-  scanast:
-    description: run Go AST security scan
-    after:
-      - toolchain
-    watches:
-      - '**/**/*.go'
-      - './support/scripts/check.sh'
-    cmds:
-      - ./support/scripts/check.sh scanast
-
-  check:
-    description: run all code checks
-    after:
-      - checkfmt
-      - lint
-      - scanast
-
-settings:
-  default-targets:
-    - test
-  docker:
-    image: 'vmware/go-kcl-toolchain:0.1.4'
-    src-volume: /go/src/github.com/vmware/vmware-go-kcl

Makefile

@@ -0,0 +1,59 @@
+.PHONY: help
+help: ## - Show this help message
+	@printf "\033[32m\xE2\x9c\x93 usage: make [target]\n\n\033[0m"
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
+.PHONY: up
+up: ## - start docker compose
+	@ cd _support/docker && docker-compose -f docker-compose.yml up
+
+.PHONY: build-common
+build-common: ## - execute build common tasks clean and mod tidy
+	@ go version
+	@ go clean
+	@ go mod download && go mod tidy
+	@ go mod verify
+
+.PHONY: build
+build: build-common ## - build a debug binary to the current platform (windows, linux or darwin(mac))
+	@ echo building
+	@ go build -v ./...
+	@ echo "done"
+
+.PHONY: format-check
+format-check: ## - check files format using gofmt
+	@ ./_support/scripts/ci.sh fmtcheck
+
+.PHONY: format-check
+format: ## - apply golang file format using gofmt
+	@ ./_support/scripts/ci.sh format
+
+.PHONY: test
+test: build-common ## - execute go test command for unit and mocked tests
+	@ ./_support/scripts/ci.sh unittest
+
+.PHONY: integration-test
+integration-test: ## - execute go test command for integration tests (aws credentials needed)
+	@ go test -v -cover -race ./test
+
+.PHONY: scan
+scan: ## - execute static code analysis
+	@ ./_support/scripts/ci.sh scan
+
+.PHONY: lint
+lint: ## - runs golangci-lint
+	@ ./_support/scripts/ci.sh lint
+
+.PHONY: ci-lint-docker
+lint-docker: ## - runs golangci-lint with docker container
+	@ docker run --rm -v "$(shell pwd)":/app -w /app ${LINT_IMAGE} golangci-lint run ${LINT_FLAGS}
+
+.PHONY: sonar-scan
+sonar-scan: ## - start sonar qube locally with docker (you will need docker installed in your machine)
+	@ # after start, setup a new project with the name sms-local and a new token sms-token, fill the token against the -Dsonar.login= parameter.
+	@ # login with user: admin pwd: vmware
+	@ $(SHELL) _support/scripts/sonar-scan.sh
+
+.PHONY: sonar-stop
+sonar-stop: ## - stop sonar qube docker container
+	@ docker stop sonarqube

_support/scripts/ci.sh

@@ -0,0 +1,105 @@
+#!/usr/bin/env bash
+
+function local_go_pkgs() {
+    find './clientlibrary' -name '*.go' | \
+        grep -Fv '/vendor/' | \
+        grep -Fv '/go/' | \
+        grep -Fv '/gen/' | \
+        grep -Fv '/tmp/' | \
+        grep -Fv '/run/' | \
+        grep -Fv '/tests/' | \
+        sed -r 's|(.+)/[^/]+\.go$|\1|g' | \
+        sort -u
+}
+
+function checkfmt() {
+    local files=""
+    files="$(find . -type f -iname "*.go" -exec gofmt -l {} \;)"
+
+    if [ -n "$files" ]; then
+        echo "You need to run \"gofmt -w ./\" to fix your formatting."
+        echo "$files" >&2
+        return 1
+    fi
+}
+
+function go_format() {
+    echo "go formatting..."
+    gofmt -w ./
+    echo "done"
+}
+
+function lint() {
+  #	golangci-lint run --enable-all -D forbidigo -D gochecknoglobals -D gofumpt -D gofmt -D nlreturn
+    golangci-lint run \
+        --skip-files=_mock.go \
+        --disable=golint \
+        --skip-dirs=test \
+        --fast \
+        --timeout=600s \
+        --verbose \
+        "$(local_go_pkgs)"
+}
+
+function test() {
+  	go list ./... | grep -v /test | \
+   	xargs -L 1 -I% bash -c 'echo -e "\n**************** Package: % ****************" && go test % -v -cover -race ./...'
+}
+
+function scanast() {
+    gosec version
+    gosec ./... > security.log 2>&1
+
+    local issues=""
+    issues=$(grep -c 'Severity: MEDIUM' security.log | grep -v deaggregator | grep -c _)
+    if [ -n "$issues" ] && [ "$issues" -gt 0 ]; then
+        echo ""
+        echo "Medium Severity Issues:"
+        grep -e "Severity: MEDIUM" -A 1 security.log
+        echo "$issues" "medium severity issues found."
+    fi
+
+    local issues=""
+    local issues_count=""
+    issues="$(grep -E 'Severity: HIGH' security.log | grep -v vendor)"
+    issues_count="$(grep -E 'Severity: HIGH' security.log | grep -v vendor | grep -c _)"
+    if [ -n "$issues_count" ] && [ "$issues_count" -gt 0 ]; then
+        echo ""
+        echo "High Severity Issues:"
+        grep -E "Severity: HIGH" -A 1 security.log
+        echo "$issues_count" "high severity issues found."
+        echo "$issues"
+        echo "You need to resolve the high severity issues at the least."
+        exit 1
+    fi
+
+    local issues=""
+    local issues_count=""
+    issues="$(grep -E 'Errors unhandled' security.log | grep -v vendor | grep -v /src/go/src)"
+    issues_count="$(grep -E 'Errors unhandled' security.log | grep -v vendor | grep -v /src/go/src | grep -c _)"
+    if [ -n "$issues_count" ] && [ "$issues_count" -gt 0 ]; then
+        echo ""
+        echo "Unhandled errors:"
+        grep -E "Errors unhandled" security.log
+        echo "$issues_count" "unhandled errors, please indicate with the right comment that this case is ok, or handle the error."
+        echo "$issues"
+        echo "You need to resolve the all unhandled errors."
+        exit 1
+    fi
+
+    rm -f security.log
+}
+
+function usage() {
+    echo "check.sh fmt|lint" >&2
+    exit 2
+}
+
+case "$1" in
+    fmtcheck) checkfmt ;;
+    format) go_format ;;
+    lint) lint ;;
+    unittest) test ;;
+    scan) scanast ;;
+    *) usage ;;
+esac

_support/scripts/sonar-scan.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+
+########################
+# requirements:        #
+# 0. docker            #
+# 1. wget              #
+# 2. curl              #
+# 3. jq                #
+# 4. sonar-scanner     #
+########################
+
+set -e
+
+projectKey="vmware-go-kcl-v2"
+user_tokenName="local_token"
+username="admin"
+user_password="admin"
+new_password="vmware"
+url="http://localhost"
+port="9000"
+
+if [[ "$( docker container inspect -f '{{.State.Running}}' sonarqube )" == "true" ]];
+then
+  docker ps
+else
+  docker run --rm -d --name sonarqube -e SONAR_ES_BOOTSTRAP_CHECKS_DISABLE=true -p 9000:9000 sonarqube
+fi
+
+echo "waiting for sonarqube starts..."
+wget -q -O - "$@" http://localhost:9000 | awk '/STARTING/{ print $0 }' | xargs
+
+STATUS="$(wget -q -O - "$@" http://localhost:9000 | awk '/UP/{ print $0 }')"
+while [ -z "$STATUS" ]
+do
+	sleep 2
+	STATUS="$(wget -q -O - "$@" http://localhost:9000 | awk '/UP/{ print $0 }')"
+	printf "."
+done
+
+printf '\n %s' "${STATUS}" | xargs
+echo ""
+
+# change the default password to avoid create a new one when login for the very first time
+curl -u ${username}:${user_password} -X POST "${url}:${port}/api/users/change_password?login=${username}&previousPassword=${user_password}&password=${new_password}"
+
+# search the specific user tokens for SonarQube
+hasToken=$(curl --silent -u ${username}:${new_password} -X GET "${url}:${port}/api/user_tokens/search")
+if [[ -n "${hasToken}"  ]]; then
+  # Revoke the user token for SonarQube
+  curl -X POST -H "Content-Type: application/x-www-form-urlencoded" -d "name=${user_tokenName}" -u ${username}:${new_password} "${url}:${port}"/api/user_tokens/revoke
+fi
+
+# generate new token
+token=$(curl --silent -X POST -H "Content-Type: application/x-www-form-urlencoded" -d "name=${user_tokenName}" -u ${username}:${new_password} "${url}:${port}"/api/user_tokens/generate | jq '.token' | xargs)
+
+# scan and push the results to localhost docker container
+sonar-scanner -Dsonar.projectKey="${projectKey}" \
+              -Dsonar.projectName="${projectKey}" \
+              -Dsonar.sources=. \
+              -Dsonar.exclusions="internal/records/**, test/**" \
+              -Dsonar.host.url="${url}:${port}" \
+              -Dsonar.login="${token}"
+

go.sum

@@ -383,8 +383,6 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211106132015-ebca88c72f68 h1:Ywe/f3fNleF8I6F6qv3MeFoSZ6CTf2zBMMa/7qVML8M=
-golang.org/x/sys v0.0.0-20211106132015-ebca88c72f68/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211107104306-e0b2ad06fe42 h1:G2DDmludOQZoWbpCr7OKDxnl478ZBGMcOhrv+ooX/Q4=
 golang.org/x/sys v0.0.0-20211107104306-e0b2ad06fe42/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

support/scripts/check.sh

@@ -1,75 +0,0 @@
-#!/usr/bin/env bash
-
-. support/scripts/functions.sh
-
-checkfmt() {
-    local files="$(gofmt -l $(local_go_pkgs))"
-    if [ -n "$files" ]; then
-        echo "You need to run \"gofmt -w ./\" to fix your formating."
-        echo "$files" >&2
-        return 1
-    fi
-}
-
-lint() {
-    golangci-lint run \
-        --skip-files=_mock.go \
-        --disable=golint \
-        --skip-dirs=test \
-        --fast \
-        --timeout=600s \
-        --verbose \
-        $(local_go_pkgs)
-}
-
-scanast() {
-    set +e
-    gosec version
-    gosec ./... > security.log 2>&1
-    set -e
-
-    local issues="$(grep -E 'Severity: MEDIUM' security.log | wc -l)"
-    if [ -n $issues ] && [ $issues -gt 0 ]; then
-        echo ""
-        echo "Medium Severity Issues:"
-        grep -E "Severity: MEDIUM" -A 1 security.log
-        echo $issues "medium severity issues found."
-    fi
-
-    local issues="$(grep -E 'Severity: HIGH' security.log | grep -v vendor)"
-    local issues_count="$(grep -E 'Severity: HIGH' security.log | grep -v vendor | wc -l)"
-    if [ -n $issues_count ] && [ $issues_count -gt 0 ]; then
-        echo ""
-        echo "High Severity Issues:"
-        grep -E "Severity: HIGH" -A 1 security.log
-        echo $issues_count "high severity issues found."
-        echo $issues
-        echo "You need to resolve the high severity issues at the least."
-        exit 1
-    fi
-
-    local issues="$(grep -E 'Errors unhandled' security.log | grep -v vendor | grep -v /src/go/src)"
-    local issues_count="$(grep -E 'Errors unhandled' security.log | grep -v vendor | grep -v /src/go/src | wc -l)"
-    if [ -n $issues_count ] && [ $issues_count -gt 0 ]; then
-        echo ""
-        echo "Unhandled errors:"
-        grep -E "Errors unhandled" security.log
-        echo $issues_count "unhandled errors, please indicate with the right comment that this case is ok, or handle the error."
-        echo $issues
-        echo "You need to resolve the all unhandled errors."
-        exit 1
-    fi
-    rm security.log
-}
-
-usage() {
-    echo "check.sh fmt|lint" >&2
-    exit 2
-}
-
-case "$1" in
-    fmt) checkfmt ;;
-    lint) lint ;;
-    scanast) scanast;;
-    *) usage ;;
-esac

support/scripts/ci.sh

@@ -1,7 +0,0 @@
-#!/bin/bash
-
-# Run only the integration tests
-# go test -race ./test
-echo "Warning: Cannot find a good way to inject AWS credential to hmake container"
-echo "Don't use hmake ci. Use the following command directly"
-echo "go test -race ./test"

support/scripts/functions.sh

@@ -1,59 +0,0 @@
-set -ex
-
-# PROJ_ROOT specifies the project root
-export PROJ_ROOT="$HMAKE_PROJECT_DIR"
-
-# Add /go in GOPATH because that's the original GOPATH in toolchain
-export GOPATH=/go:$PROJ_ROOT
-
-local_go_pkgs() {
-    find './clientlibrary/' -name '*.go' | \
-        grep -Fv '/vendor/' | \
-        grep -Fv '/go/' | \
-        grep -Fv '/gen/' | \
-        grep -Fv '/tmp/' | \
-        grep -Fv '/run/' | \
-        grep -Fv '/tests/' | \
-        sed -r 's|(.+)/[^/]+\.go$|\1|g' | \
-        sort -u
-}
-
-version_suffix() {
-    local suffix=$(git log -1 --format=%h 2>/dev/null || true)
-    if [ -n "$suffix" ]; then
-        test -z "$(git status --porcelain 2>/dev/null || true)" || suffix="${suffix}+"
-        echo -n "-g${suffix}"
-    else
-        echo -n -dev
-    fi
-}
-
-git_commit_hash() {
-	echo $(git rev-parse --short HEAD)
-}
-
-# Due to Go plugin genhash algorithm simply takes full source path
-# from archive, it generates different plugin hash if source path of
-# shared pkg is different, and causes load failure.
-# as a workaround, lookup shared pkg and place it to fixed path
-FIX_GOPATH=/tmp/go
-
-fix_go_pkg() {
-    local pkg="$1" base
-    for p in ${GOPATH//:/ }; do
-        if [ -d "$p/src/$pkg" ]; then
-            base="$p"
-            break
-        fi
-    done
-
-    if [ -z "$base" ]; then
-        echo "Package $pkg not found in GOPATH: $GOPATH" >&2
-        return 1
-    fi
-
-    local fix_pkg_path="$FIX_GOPATH/src/$pkg"
-    rm -f "$fix_pkg_path"
-    mkdir -p "$(dirname $fix_pkg_path)"
-    ln -s "$base/src/$pkg" "$fix_pkg_path"
-}

support/scripts/test.sh

@@ -1,5 +0,0 @@
-#!/bin/bash
-. support/scripts/functions.sh
-
-# Run only the unit tests and not integration tests
-go test -cover -race $(local_go_pkgs)

support/toolchain/HyperMake

@@ -1,28 +0,0 @@
----
-format: hypermake.v0
-
-name: go-kcl
-description: VMWare Go-KCL Amazon Kinesis Client Library in Go
-
-targets:
-  rebuild-toolchain:
-    description: build toolchain image
-    watches:
-      - docker
-    build: docker
-    cache: false
-    tags:
-      - vmware/go-kcl-toolchain:latest
-
-  push-toolchain:
-    description: push toolchain image
-    after:
-      - rebuild-toolchain
-    push:
-      - vmware/go-kcl-toolchain:latest
-
-settings:
-  default-targets:
-    - rebuild-toolchain
-  docker:
-    image: 'vmware/go-kcl-toolchain:0.1.4'

support/toolchain/docker/Dockerfile

@@ -1,8 +0,0 @@
-FROM golang:1.17
-ENV PATH /go/bin:/src/bin:/root/go/bin:/usr/local/go/bin:$PATH
-ENV GOPATH /go:/src
-RUN go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.42.1 && \
-    go install golang.org/x/tools/cmd/...@latest && \
-    go install github.com/go-delve/delve/cmd/dlv@latest && \
-    curl -sfL https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s v2.8.1 && \
-    chmod -R a+rw /go