diff --git a/.gitignore b/.gitignore index 5e7d273..79c027d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,21 @@ * # Except this file !.gitignore +======= +/src/gen +/src/vendor +!/src/vendor/manifest +/bin +/pkg +/tmp +/log +/vms +/run +/go +.hmake +.hmakerc +.project +.idea +.vscode +*_mock_test.go +filenames diff --git a/.gitreview b/.gitreview new file mode 100644 index 0000000..ef3128e --- /dev/null +++ b/.gitreview @@ -0,0 +1,5 @@ +[gerrit] +host=review.ec.eng.vmware.com +port=29418 +project=cascade-kinesis-client +defaultbranch=develop diff --git a/HyperMake b/HyperMake new file mode 100644 index 0000000..b966419 --- /dev/null +++ b/HyperMake @@ -0,0 +1,99 @@ +--- +format: hypermake.v0 + +name: cascade-kinesis-client +description: Kinesis Client in Go + +targets: + rebuild-toolchain: + description: build toolchain image + watches: + - support/docker/toolchain + build: support/docker/toolchain + + toolchain: + description: placeholder for additional toolchain dependencies + + deps: + description: setup dependencies + after: + - 'deps-*' + + gen: + description: generate source code + after: + - 'gen-*' + + build: + description: build source code + after: + - 'build-*' + + test: + description: run unit tests + after: + - deps + - gen + always: true + cmds: + - ./support/scripts/test.sh + + ci: + description: run CI tests + after: + - check + cmds: + - ./support/scripts/ci.sh + + checkfmt: + description: check code format + after: + - toolchain + watches: + - support/scripts/check.sh + always: true + cmds: + - ./support/scripts/check.sh fmt + + lint: + description: run lint to check code + after: + - toolchain + watches: + - support/scripts/check.sh + always: true + cmds: + - ./support/scripts/check.sh lint + + scanast: + description: run Go AST security scan + after: + - toolchain + watches: + - '**/**/*.go' + - './support/scripts/check.sh' + cmds: + - ./support/scripts/check.sh scanast + + check: + description: run all code checks + after: + - checkfmt + - lint + + deps-kcl: + description: populate vendor packages + after: + - toolchain + watches: + - src/vendor/manifest + workdir: src + cmds: + - gvt restore + +settings: + default-targets: + - ci + docker: + image: 'vmware/cascade-kcl-toolchain:latest' + src-volume: /home/cascade-kinesis-client diff --git a/README.md b/README.md index 9db664b..57cd94a 100644 --- a/README.md +++ b/README.md @@ -27,3 +27,9 @@ as an open-source patch. For more detailed information, refer to [CONTRIBUTING.m ## License +======= +# Cascade Kinesis Client Library for GO + +The **Cascade Kinesis Client Library for GO** (Cascade KCL) enables Go developers to easily consume and process data from [Amazon Kinesis][kinesis]. + +It is a re-implementation on Amazon's Kinesis Client Library in pure Go without using KCL's multi-language support. diff --git a/src/vendor/manifest b/src/vendor/manifest new file mode 100644 index 0000000..9d112c1 --- /dev/null +++ b/src/vendor/manifest @@ -0,0 +1,167 @@ +{ + "version": 0, + "dependencies": [ + { + "importpath": "github.com/aws/aws-sdk-go", + "repository": "https://github.com/aws/aws-sdk-go", + "vcs": "git", + "revision": "365b4d34369496e650e3056b33fce4e1a25cfc72", + "branch": "master", + "notests": true + }, + { + "importpath": "github.com/asaskevich/govalidator", + "repository": "https://github.com/asaskevich/govalidator", + "vcs": "git", + "revision": "38ddb4612a5dfc2878731749ee825853d9f0aaa1", + "branch": "master", + "notests": true + }, + { + "importpath": "github.com/sirupsen/logrus", + "repository": "https://github.com/sirupsen/logrus", + "vcs": "git", + "revision": "51dc0fc64317a2861273909081f9c315786533eb", + "branch": "master", + "notests": true + }, + { + "importpath": "github.com/beorn7/perks/quantile", + "repository": "https://github.com/beorn7/perks", + "vcs": "git", + "revision": "4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9", + "branch": "master", + "path": "/quantile", + "notests": true + }, + { + "importpath": "gopkg.in/gemnasium/logrus-airbrake-hook.v2", + "repository": "https://gopkg.in/gemnasium/logrus-airbrake-hook.v2", + "vcs": "git", + "revision": "e928b033a891c0175fb643d5aa0779e86325eb12", + "branch": "master", + "notests": true + }, + { + "importpath": "github.com/matttproud/golang_protobuf_extensions/pbutil", + "repository": "https://github.com/matttproud/golang_protobuf_extensions", + "vcs": "git", + "revision": "c12348ce28de40eed0136aa2b644d0ee0650e56c", + "branch": "master", + "path": "/pbutil", + "notests": true + }, + { + "importpath": "github.com/prometheus/client_golang/prometheus", + "repository": "https://github.com/prometheus/client_golang", + "vcs": "git", + "revision": "26b897001974f2b4ee6688377873e4d6f61d533c", + "branch": "master", + "path": "prometheus", + "notests": true + }, + { + "importpath": "github.com/prometheus/client_model/go", + "repository": "https://github.com/prometheus/client_model", + "vcs": "git", + "revision": "6f3806018612930941127f2a7c6c453ba2c527d2", + "branch": "master", + "path": "/go", + "notests": true + }, + { + "importpath": "github.com/prometheus/common/expfmt", + "repository": "https://github.com/prometheus/common", + "vcs": "git", + "revision": "3e6a7635bac6573d43f49f97b47eb9bda195dba8", + "branch": "master", + "path": "/expfmt", + "notests": true + }, + { + "importpath": "github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg", + "repository": "https://github.com/prometheus/common", + "vcs": "git", + "revision": "3e6a7635bac6573d43f49f97b47eb9bda195dba8", + "branch": "master", + "path": "internal/bitbucket.org/ww/goautoneg", + "notests": true + }, + { + "importpath": "github.com/prometheus/common/model", + "repository": "https://github.com/prometheus/common", + "vcs": "git", + "revision": "3e6a7635bac6573d43f49f97b47eb9bda195dba8", + "branch": "master", + "path": "model", + "notests": true + }, + { + "importpath": "github.com/astaxie/beego/cache", + "repository": "https://github.com/astaxie/beego", + "vcs": "git", + "revision": "a7354d2d084003e4122d6e69f7e5ab594fd117b2", + "branch": "master", + "path": "cache", + "notests": true + }, + { + "importpath": "github.com/prometheus/procfs", + "repository": "https://github.com/prometheus/procfs", + "vcs": "git", + "revision": "e645f4e5aaa8506fc71d6edbc5c4ff02c04c46f2", + "branch": "master", + "notests": true + }, + { + "importpath": "github.com/golang/protobuf/proto", + "repository": "https://github.com/golang/protobuf", + "vcs": "git", + "revision": "6a1fa9404c0aebf36c879bc50152edcc953910d2", + "branch": "master", + "path": "/proto", + "notests": true + }, + { + "importpath": "github.com/golang/protobuf/ptypes/any", + "repository": "https://github.com/golang/protobuf", + "vcs": "git", + "revision": "6a1fa9404c0aebf36c879bc50152edcc953910d2", + "branch": "master", + "path": "ptypes/any", + "notests": true + }, + { + "importpath": "github.com/google/uuid", + "repository": "https://github.com/google/uuid", + "vcs": "git", + "revision": "6a5e28554805e78ea6141142aba763936c4761c0", + "branch": "master", + "notests": true + }, + { + "importpath": "github.com/matryer/try", + "repository": "https://github.com/matryer/try", + "vcs": "git", + "revision": "312d2599e12e89ca89b52a09597394f449235d80", + "branch": "master", + "notests": true + }, + { + "importpath": "github.com/stretchr/testify", + "repository": "https://github.com/stretchr/testify", + "vcs": "git", + "revision": "12b6f73e6084dad08a7c6e575284b177ecafbc71", + "branch": "master", + "notests": true + }, + { + "importpath": "gopkg.in/yaml.v2", + "repository": "https://gopkg.in/yaml.v2", + "vcs": "git", + "revision": "cd8b52f8269e0feb286dfeef29f8fe4d5b397e0b", + "branch": "v2", + "notests": true + } + ] +} diff --git a/support/scripts/check.sh b/support/scripts/check.sh new file mode 100755 index 0000000..fbab703 --- /dev/null +++ b/support/scripts/check.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +. support/scripts/functions.sh + +checkfmt() { + local files="$(gofmt -l $(local_go_pkgs))" + if [ -n "$files" ]; then + echo "You need to run \"gofmt -w ./\" to fix your formating." + echo "$files" >&2 + return 1 + fi +} + +lint() { + gometalinter \ + --exclude=_mock.go \ + --disable=gotype \ + --vendor \ + --skip=test \ + --fast \ + --deadline=600s \ + --severity=golint:error \ + --errors \ + $(local_go_pkgs) +} + +scanast() { + set +e + gas ./... > security.log 2>&1 + set -e + + local issues=$(grep -E "Severity: MEDIUM" security.log | wc -l) + if [ -n $issues ] && [ $issues -gt 0 ]; then + echo "" + echo "Medium Severity Issues:" + grep -E "Severity: MEDIUM" -A 1 security.log + echo $issues "medium severity issues found." + fi + + local issues=$(grep -E "Severity: HIGH" security.log | grep -v "vendor") + local issues_count=$(grep -E "Severity: HIGH" security.log | grep -v "vendor" | wc -l) + if [ -n $issues_count ] && [ $issues_count -gt 0 ]; then + echo "" + echo "High Severity Issues:" + grep -E "Severity: HIGH" -A 1 security.log + echo $issues_count "high severity issues found." + echo $issues + echo "You need to resolve the high severity issues at the least." + exit 1 + fi + + local issues=$(grep -E "Errors unhandled" security.log | grep -v "vendor" | grep -v "/src/go/src") + local issues_count=$(grep -E "Errors unhandled" security.log | grep -v "vendor" | grep -v "/src/go/src" | wc -l) + if [ -n $issues_count ] && [ $issues_count -gt 0 ]; then + echo "" + echo "Unhandled errors:" + grep -E "Errors unhandled" security.log + echo $issues_count "unhandled errors, please indicate with the right comment that this case is ok, or handle the error." + echo $issues + echo "You need to resolve the all unhandled errors." + exit 1 + fi + rm security.log +} + +usage() { + echo "check.sh fmt|lint" >&2 + exit 2 +} + +case "$1" in + fmt) checkfmt ;; + lint) lint ;; + scanast) scanast;; + *) usage ;; +esac diff --git a/support/scripts/ci.sh b/support/scripts/ci.sh new file mode 100755 index 0000000..012ea40 --- /dev/null +++ b/support/scripts/ci.sh @@ -0,0 +1,2 @@ +#!/bin/bash +./support/scripts/test.sh diff --git a/support/scripts/functions.sh b/support/scripts/functions.sh new file mode 100644 index 0000000..c76d266 --- /dev/null +++ b/support/scripts/functions.sh @@ -0,0 +1,65 @@ +set -ex + +# PROJ_ROOT specifies the project root +export PROJ_ROOT="$HMAKE_PROJECT_DIR" + +# Add /go in GOPATH because that's the original GOPATH in toolchain +export GOPATH=/go:$PROJ_ROOT + +local_go_pkgs() { + find . -name '*.go' | \ + grep -Fv '/vendor/' | \ + grep -Fv '/go/' | \ + grep -Fv '/gen/' | \ + grep -Fv '/tmp/' | \ + grep -Fv '/run/' | \ + grep -Fv '/tests/' | \ + sort -u +} + +local_test_pkgs() { + find ./src/test -name '*.go' | \ + grep -Fv '_test.go' | \ + sed -r 's|(.+)/[^/]+\.go$|\1|g' | \ + sort -u +} + +version_suffix() { + local suffix=$(git log -1 --format=%h 2>/dev/null || true) + if [ -n "$suffix" ]; then + test -z "$(git status --porcelain 2>/dev/null || true)" || suffix="${suffix}+" + echo -n "-g${suffix}" + else + echo -n -dev + fi +} + +git_commit_hash() { + echo $(git rev-parse --short HEAD) +} + +# Due to Go plugin genhash algorithm simply takes full source path +# from archive, it generates different plugin hash if source path of +# shared pkg is different, and causes load failure. +# as a workaround, lookup shared pkg and place it to fixed path +FIX_GOPATH=/tmp/go + +fix_go_pkg() { + local pkg="$1" base + for p in ${GOPATH//:/ }; do + if [ -d "$p/src/$pkg" ]; then + base="$p" + break + fi + done + + if [ -z "$base" ]; then + echo "Package $pkg not found in GOPATH: $GOPATH" >&2 + return 1 + fi + + local fix_pkg_path="$FIX_GOPATH/src/$pkg" + rm -f "$fix_pkg_path" + mkdir -p "$(dirname $fix_pkg_path)" + ln -s "$base/src/$pkg" "$fix_pkg_path" +} diff --git a/support/scripts/test.sh b/support/scripts/test.sh new file mode 100755 index 0000000..78c0986 --- /dev/null +++ b/support/scripts/test.sh @@ -0,0 +1,5 @@ +#!/bin/bash +. support/scripts/functions.sh + +# Run only the unit tests and not integration tests +go test -race $(local_go_pkgs) diff --git a/support/toolchain/HyperMake b/support/toolchain/HyperMake new file mode 100644 index 0000000..708ccf3 --- /dev/null +++ b/support/toolchain/HyperMake @@ -0,0 +1,28 @@ +--- +format: hypermake.v0 + +name: cascade-kcl +description: Amazon Kinesis Client Library in Go + +targets: + rebuild-toolchain: + description: build toolchain image + watches: + - docker + build: docker + cache: false + tags: + - vmware/cascade-kcl-toolchain:latest + + push-toolchain: + description: push toolchain image + after: + - rebuild-toolchain + push: + - vmware/cascade-toolchain:latest + +settings: + default-targets: + - rebuild-toolchain + docker: + image: 'vmware/cascade-kcl-toolchain:0.0.0' diff --git a/support/toolchain/docker/Dockerfile b/support/toolchain/docker/Dockerfile new file mode 100644 index 0000000..764f92f --- /dev/null +++ b/support/toolchain/docker/Dockerfile @@ -0,0 +1,10 @@ +FROM golang:1.10 +ENV PATH /go/bin:/src/bin:/root/go/bin:/usr/local/go/bin:$PATH +ENV GOPATH /go:/src +RUN go get -v github.com/alecthomas/gometalinter && \ + go get -v golang.org/x/tools/cmd/... && \ + go get -v github.com/FiloSottile/gvt && \ + go get -v github.com/GoASTScanner/gas/cmd/gas/... && \ + go get github.com/derekparker/delve/cmd/dlv && \ + gometalinter --install && \ + chmod -R a+rw /go \ No newline at end of file