CKL: Initial version for starting repository
Change-Id: Ic8dc81a437d9c72e841048359745e245c6c4895b
This commit is contained in:
Tao Jiang
parent
f7763e6d8b
commit
7b35571d9e
11 changed files with 481 additions and 0 deletions
18
.gitignore
vendored
18
.gitignore
vendored
|
|
@ -2,3 +2,21 @@
|
||||||
*
|
*
|
||||||
# Except this file
|
# Except this file
|
||||||
!.gitignore
|
!.gitignore
|
||||||
|
=======
|
||||||
|
/src/gen
|
||||||
|
/src/vendor
|
||||||
|
!/src/vendor/manifest
|
||||||
|
/bin
|
||||||
|
/pkg
|
||||||
|
/tmp
|
||||||
|
/log
|
||||||
|
/vms
|
||||||
|
/run
|
||||||
|
/go
|
||||||
|
.hmake
|
||||||
|
.hmakerc
|
||||||
|
.project
|
||||||
|
.idea
|
||||||
|
.vscode
|
||||||
|
*_mock_test.go
|
||||||
|
filenames
|
||||||
|
|
|
||||||
5
.gitreview
Normal file
5
.gitreview
Normal file
|
|
@ -0,0 +1,5 @@
|
||||||
|
[gerrit]
|
||||||
|
host=review.ec.eng.vmware.com
|
||||||
|
port=29418
|
||||||
|
project=cascade-kinesis-client
|
||||||
|
defaultbranch=develop
|
||||||
99
HyperMake
Normal file
99
HyperMake
Normal file
|
|
@ -0,0 +1,99 @@
|
||||||
|
---
|
||||||
|
format: hypermake.v0
|
||||||
|
|
||||||
|
name: cascade-kinesis-client
|
||||||
|
description: Kinesis Client in Go
|
||||||
|
|
||||||
|
targets:
|
||||||
|
rebuild-toolchain:
|
||||||
|
description: build toolchain image
|
||||||
|
watches:
|
||||||
|
- support/docker/toolchain
|
||||||
|
build: support/docker/toolchain
|
||||||
|
|
||||||
|
toolchain:
|
||||||
|
description: placeholder for additional toolchain dependencies
|
||||||
|
|
||||||
|
deps:
|
||||||
|
description: setup dependencies
|
||||||
|
after:
|
||||||
|
- 'deps-*'
|
||||||
|
|
||||||
|
gen:
|
||||||
|
description: generate source code
|
||||||
|
after:
|
||||||
|
- 'gen-*'
|
||||||
|
|
||||||
|
build:
|
||||||
|
description: build source code
|
||||||
|
after:
|
||||||
|
- 'build-*'
|
||||||
|
|
||||||
|
test:
|
||||||
|
description: run unit tests
|
||||||
|
after:
|
||||||
|
- deps
|
||||||
|
- gen
|
||||||
|
always: true
|
||||||
|
cmds:
|
||||||
|
- ./support/scripts/test.sh
|
||||||
|
|
||||||
|
ci:
|
||||||
|
description: run CI tests
|
||||||
|
after:
|
||||||
|
- check
|
||||||
|
cmds:
|
||||||
|
- ./support/scripts/ci.sh
|
||||||
|
|
||||||
|
checkfmt:
|
||||||
|
description: check code format
|
||||||
|
after:
|
||||||
|
- toolchain
|
||||||
|
watches:
|
||||||
|
- support/scripts/check.sh
|
||||||
|
always: true
|
||||||
|
cmds:
|
||||||
|
- ./support/scripts/check.sh fmt
|
||||||
|
|
||||||
|
lint:
|
||||||
|
description: run lint to check code
|
||||||
|
after:
|
||||||
|
- toolchain
|
||||||
|
watches:
|
||||||
|
- support/scripts/check.sh
|
||||||
|
always: true
|
||||||
|
cmds:
|
||||||
|
- ./support/scripts/check.sh lint
|
||||||
|
|
||||||
|
scanast:
|
||||||
|
description: run Go AST security scan
|
||||||
|
after:
|
||||||
|
- toolchain
|
||||||
|
watches:
|
||||||
|
- '**/**/*.go'
|
||||||
|
- './support/scripts/check.sh'
|
||||||
|
cmds:
|
||||||
|
- ./support/scripts/check.sh scanast
|
||||||
|
|
||||||
|
check:
|
||||||
|
description: run all code checks
|
||||||
|
after:
|
||||||
|
- checkfmt
|
||||||
|
- lint
|
||||||
|
|
||||||
|
deps-kcl:
|
||||||
|
description: populate vendor packages
|
||||||
|
after:
|
||||||
|
- toolchain
|
||||||
|
watches:
|
||||||
|
- src/vendor/manifest
|
||||||
|
workdir: src
|
||||||
|
cmds:
|
||||||
|
- gvt restore
|
||||||
|
|
||||||
|
settings:
|
||||||
|
default-targets:
|
||||||
|
- ci
|
||||||
|
docker:
|
||||||
|
image: 'vmware/cascade-kcl-toolchain:latest'
|
||||||
|
src-volume: /home/cascade-kinesis-client
|
||||||
|
|
@ -27,3 +27,9 @@ as an open-source patch. For more detailed information, refer to [CONTRIBUTING.m
|
||||||
|
|
||||||
## License
|
## License
|
||||||
|
|
||||||
|
=======
|
||||||
|
# Cascade Kinesis Client Library for GO
|
||||||
|
|
||||||
|
The **Cascade Kinesis Client Library for GO** (Cascade KCL) enables Go developers to easily consume and process data from [Amazon Kinesis][kinesis].
|
||||||
|
|
||||||
|
It is a re-implementation on Amazon's Kinesis Client Library in pure Go without using KCL's multi-language support.
|
||||||
|
|
|
||||||
167
src/vendor/manifest
vendored
Normal file
167
src/vendor/manifest
vendored
Normal file
|
|
@ -0,0 +1,167 @@
|
||||||
|
{
|
||||||
|
"version": 0,
|
||||||
|
"dependencies": [
|
||||||
|
{
|
||||||
|
"importpath": "github.com/aws/aws-sdk-go",
|
||||||
|
"repository": "https://github.com/aws/aws-sdk-go",
|
||||||
|
"vcs": "git",
|
||||||
|
"revision": "365b4d34369496e650e3056b33fce4e1a25cfc72",
|
||||||
|
"branch": "master",
|
||||||
|
"notests": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"importpath": "github.com/asaskevich/govalidator",
|
||||||
|
"repository": "https://github.com/asaskevich/govalidator",
|
||||||
|
"vcs": "git",
|
||||||
|
"revision": "38ddb4612a5dfc2878731749ee825853d9f0aaa1",
|
||||||
|
"branch": "master",
|
||||||
|
"notests": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"importpath": "github.com/sirupsen/logrus",
|
||||||
|
"repository": "https://github.com/sirupsen/logrus",
|
||||||
|
"vcs": "git",
|
||||||
|
"revision": "51dc0fc64317a2861273909081f9c315786533eb",
|
||||||
|
"branch": "master",
|
||||||
|
"notests": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"importpath": "github.com/beorn7/perks/quantile",
|
||||||
|
"repository": "https://github.com/beorn7/perks",
|
||||||
|
"vcs": "git",
|
||||||
|
"revision": "4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9",
|
||||||
|
"branch": "master",
|
||||||
|
"path": "/quantile",
|
||||||
|
"notests": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"importpath": "gopkg.in/gemnasium/logrus-airbrake-hook.v2",
|
||||||
|
"repository": "https://gopkg.in/gemnasium/logrus-airbrake-hook.v2",
|
||||||
|
"vcs": "git",
|
||||||
|
"revision": "e928b033a891c0175fb643d5aa0779e86325eb12",
|
||||||
|
"branch": "master",
|
||||||
|
"notests": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"importpath": "github.com/matttproud/golang_protobuf_extensions/pbutil",
|
||||||
|
"repository": "https://github.com/matttproud/golang_protobuf_extensions",
|
||||||
|
"vcs": "git",
|
||||||
|
"revision": "c12348ce28de40eed0136aa2b644d0ee0650e56c",
|
||||||
|
"branch": "master",
|
||||||
|
"path": "/pbutil",
|
||||||
|
"notests": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"importpath": "github.com/prometheus/client_golang/prometheus",
|
||||||
|
"repository": "https://github.com/prometheus/client_golang",
|
||||||
|
"vcs": "git",
|
||||||
|
"revision": "26b897001974f2b4ee6688377873e4d6f61d533c",
|
||||||
|
"branch": "master",
|
||||||
|
"path": "prometheus",
|
||||||
|
"notests": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"importpath": "github.com/prometheus/client_model/go",
|
||||||
|
"repository": "https://github.com/prometheus/client_model",
|
||||||
|
"vcs": "git",
|
||||||
|
"revision": "6f3806018612930941127f2a7c6c453ba2c527d2",
|
||||||
|
"branch": "master",
|
||||||
|
"path": "/go",
|
||||||
|
"notests": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"importpath": "github.com/prometheus/common/expfmt",
|
||||||
|
"repository": "https://github.com/prometheus/common",
|
||||||
|
"vcs": "git",
|
||||||
|
"revision": "3e6a7635bac6573d43f49f97b47eb9bda195dba8",
|
||||||
|
"branch": "master",
|
||||||
|
"path": "/expfmt",
|
||||||
|
"notests": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"importpath": "github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg",
|
||||||
|
"repository": "https://github.com/prometheus/common",
|
||||||
|
"vcs": "git",
|
||||||
|
"revision": "3e6a7635bac6573d43f49f97b47eb9bda195dba8",
|
||||||
|
"branch": "master",
|
||||||
|
"path": "internal/bitbucket.org/ww/goautoneg",
|
||||||
|
"notests": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"importpath": "github.com/prometheus/common/model",
|
||||||
|
"repository": "https://github.com/prometheus/common",
|
||||||
|
"vcs": "git",
|
||||||
|
"revision": "3e6a7635bac6573d43f49f97b47eb9bda195dba8",
|
||||||
|
"branch": "master",
|
||||||
|
"path": "model",
|
||||||
|
"notests": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"importpath": "github.com/astaxie/beego/cache",
|
||||||
|
"repository": "https://github.com/astaxie/beego",
|
||||||
|
"vcs": "git",
|
||||||
|
"revision": "a7354d2d084003e4122d6e69f7e5ab594fd117b2",
|
||||||
|
"branch": "master",
|
||||||
|
"path": "cache",
|
||||||
|
"notests": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"importpath": "github.com/prometheus/procfs",
|
||||||
|
"repository": "https://github.com/prometheus/procfs",
|
||||||
|
"vcs": "git",
|
||||||
|
"revision": "e645f4e5aaa8506fc71d6edbc5c4ff02c04c46f2",
|
||||||
|
"branch": "master",
|
||||||
|
"notests": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"importpath": "github.com/golang/protobuf/proto",
|
||||||
|
"repository": "https://github.com/golang/protobuf",
|
||||||
|
"vcs": "git",
|
||||||
|
"revision": "6a1fa9404c0aebf36c879bc50152edcc953910d2",
|
||||||
|
"branch": "master",
|
||||||
|
"path": "/proto",
|
||||||
|
"notests": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"importpath": "github.com/golang/protobuf/ptypes/any",
|
||||||
|
"repository": "https://github.com/golang/protobuf",
|
||||||
|
"vcs": "git",
|
||||||
|
"revision": "6a1fa9404c0aebf36c879bc50152edcc953910d2",
|
||||||
|
"branch": "master",
|
||||||
|
"path": "ptypes/any",
|
||||||
|
"notests": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"importpath": "github.com/google/uuid",
|
||||||
|
"repository": "https://github.com/google/uuid",
|
||||||
|
"vcs": "git",
|
||||||
|
"revision": "6a5e28554805e78ea6141142aba763936c4761c0",
|
||||||
|
"branch": "master",
|
||||||
|
"notests": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"importpath": "github.com/matryer/try",
|
||||||
|
"repository": "https://github.com/matryer/try",
|
||||||
|
"vcs": "git",
|
||||||
|
"revision": "312d2599e12e89ca89b52a09597394f449235d80",
|
||||||
|
"branch": "master",
|
||||||
|
"notests": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"importpath": "github.com/stretchr/testify",
|
||||||
|
"repository": "https://github.com/stretchr/testify",
|
||||||
|
"vcs": "git",
|
||||||
|
"revision": "12b6f73e6084dad08a7c6e575284b177ecafbc71",
|
||||||
|
"branch": "master",
|
||||||
|
"notests": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"importpath": "gopkg.in/yaml.v2",
|
||||||
|
"repository": "https://gopkg.in/yaml.v2",
|
||||||
|
"vcs": "git",
|
||||||
|
"revision": "cd8b52f8269e0feb286dfeef29f8fe4d5b397e0b",
|
||||||
|
"branch": "v2",
|
||||||
|
"notests": true
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
76
support/scripts/check.sh
Executable file
76
support/scripts/check.sh
Executable file
|
|
@ -0,0 +1,76 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
. support/scripts/functions.sh
|
||||||
|
|
||||||
|
checkfmt() {
|
||||||
|
local files="$(gofmt -l $(local_go_pkgs))"
|
||||||
|
if [ -n "$files" ]; then
|
||||||
|
echo "You need to run \"gofmt -w ./\" to fix your formating."
|
||||||
|
echo "$files" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
lint() {
|
||||||
|
gometalinter \
|
||||||
|
--exclude=_mock.go \
|
||||||
|
--disable=gotype \
|
||||||
|
--vendor \
|
||||||
|
--skip=test \
|
||||||
|
--fast \
|
||||||
|
--deadline=600s \
|
||||||
|
--severity=golint:error \
|
||||||
|
--errors \
|
||||||
|
$(local_go_pkgs)
|
||||||
|
}
|
||||||
|
|
||||||
|
scanast() {
|
||||||
|
set +e
|
||||||
|
gas ./... > security.log 2>&1
|
||||||
|
set -e
|
||||||
|
|
||||||
|
local issues=$(grep -E "Severity: MEDIUM" security.log | wc -l)
|
||||||
|
if [ -n $issues ] && [ $issues -gt 0 ]; then
|
||||||
|
echo ""
|
||||||
|
echo "Medium Severity Issues:"
|
||||||
|
grep -E "Severity: MEDIUM" -A 1 security.log
|
||||||
|
echo $issues "medium severity issues found."
|
||||||
|
fi
|
||||||
|
|
||||||
|
local issues=$(grep -E "Severity: HIGH" security.log | grep -v "vendor")
|
||||||
|
local issues_count=$(grep -E "Severity: HIGH" security.log | grep -v "vendor" | wc -l)
|
||||||
|
if [ -n $issues_count ] && [ $issues_count -gt 0 ]; then
|
||||||
|
echo ""
|
||||||
|
echo "High Severity Issues:"
|
||||||
|
grep -E "Severity: HIGH" -A 1 security.log
|
||||||
|
echo $issues_count "high severity issues found."
|
||||||
|
echo $issues
|
||||||
|
echo "You need to resolve the high severity issues at the least."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
local issues=$(grep -E "Errors unhandled" security.log | grep -v "vendor" | grep -v "/src/go/src")
|
||||||
|
local issues_count=$(grep -E "Errors unhandled" security.log | grep -v "vendor" | grep -v "/src/go/src" | wc -l)
|
||||||
|
if [ -n $issues_count ] && [ $issues_count -gt 0 ]; then
|
||||||
|
echo ""
|
||||||
|
echo "Unhandled errors:"
|
||||||
|
grep -E "Errors unhandled" security.log
|
||||||
|
echo $issues_count "unhandled errors, please indicate with the right comment that this case is ok, or handle the error."
|
||||||
|
echo $issues
|
||||||
|
echo "You need to resolve the all unhandled errors."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
rm security.log
|
||||||
|
}
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
echo "check.sh fmt|lint" >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
|
||||||
|
case "$1" in
|
||||||
|
fmt) checkfmt ;;
|
||||||
|
lint) lint ;;
|
||||||
|
scanast) scanast;;
|
||||||
|
*) usage ;;
|
||||||
|
esac
|
||||||
2
support/scripts/ci.sh
Executable file
2
support/scripts/ci.sh
Executable file
|
|
@ -0,0 +1,2 @@
|
||||||
|
#!/bin/bash
|
||||||
|
./support/scripts/test.sh
|
||||||
65
support/scripts/functions.sh
Normal file
65
support/scripts/functions.sh
Normal file
|
|
@ -0,0 +1,65 @@
|
||||||
|
set -ex
|
||||||
|
|
||||||
|
# PROJ_ROOT specifies the project root
|
||||||
|
export PROJ_ROOT="$HMAKE_PROJECT_DIR"
|
||||||
|
|
||||||
|
# Add /go in GOPATH because that's the original GOPATH in toolchain
|
||||||
|
export GOPATH=/go:$PROJ_ROOT
|
||||||
|
|
||||||
|
local_go_pkgs() {
|
||||||
|
find . -name '*.go' | \
|
||||||
|
grep -Fv '/vendor/' | \
|
||||||
|
grep -Fv '/go/' | \
|
||||||
|
grep -Fv '/gen/' | \
|
||||||
|
grep -Fv '/tmp/' | \
|
||||||
|
grep -Fv '/run/' | \
|
||||||
|
grep -Fv '/tests/' | \
|
||||||
|
sort -u
|
||||||
|
}
|
||||||
|
|
||||||
|
local_test_pkgs() {
|
||||||
|
find ./src/test -name '*.go' | \
|
||||||
|
grep -Fv '_test.go' | \
|
||||||
|
sed -r 's|(.+)/[^/]+\.go$|\1|g' | \
|
||||||
|
sort -u
|
||||||
|
}
|
||||||
|
|
||||||
|
version_suffix() {
|
||||||
|
local suffix=$(git log -1 --format=%h 2>/dev/null || true)
|
||||||
|
if [ -n "$suffix" ]; then
|
||||||
|
test -z "$(git status --porcelain 2>/dev/null || true)" || suffix="${suffix}+"
|
||||||
|
echo -n "-g${suffix}"
|
||||||
|
else
|
||||||
|
echo -n -dev
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
git_commit_hash() {
|
||||||
|
echo $(git rev-parse --short HEAD)
|
||||||
|
}
|
||||||
|
|
||||||
|
# Due to Go plugin genhash algorithm simply takes full source path
|
||||||
|
# from archive, it generates different plugin hash if source path of
|
||||||
|
# shared pkg is different, and causes load failure.
|
||||||
|
# as a workaround, lookup shared pkg and place it to fixed path
|
||||||
|
FIX_GOPATH=/tmp/go
|
||||||
|
|
||||||
|
fix_go_pkg() {
|
||||||
|
local pkg="$1" base
|
||||||
|
for p in ${GOPATH//:/ }; do
|
||||||
|
if [ -d "$p/src/$pkg" ]; then
|
||||||
|
base="$p"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ -z "$base" ]; then
|
||||||
|
echo "Package $pkg not found in GOPATH: $GOPATH" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
local fix_pkg_path="$FIX_GOPATH/src/$pkg"
|
||||||
|
rm -f "$fix_pkg_path"
|
||||||
|
mkdir -p "$(dirname $fix_pkg_path)"
|
||||||
|
ln -s "$base/src/$pkg" "$fix_pkg_path"
|
||||||
|
}
|
||||||
5
support/scripts/test.sh
Executable file
5
support/scripts/test.sh
Executable file
|
|
@ -0,0 +1,5 @@
|
||||||
|
#!/bin/bash
|
||||||
|
. support/scripts/functions.sh
|
||||||
|
|
||||||
|
# Run only the unit tests and not integration tests
|
||||||
|
go test -race $(local_go_pkgs)
|
||||||
28
support/toolchain/HyperMake
Normal file
28
support/toolchain/HyperMake
Normal file
|
|
@ -0,0 +1,28 @@
|
||||||
|
---
|
||||||
|
format: hypermake.v0
|
||||||
|
|
||||||
|
name: cascade-kcl
|
||||||
|
description: Amazon Kinesis Client Library in Go
|
||||||
|
|
||||||
|
targets:
|
||||||
|
rebuild-toolchain:
|
||||||
|
description: build toolchain image
|
||||||
|
watches:
|
||||||
|
- docker
|
||||||
|
build: docker
|
||||||
|
cache: false
|
||||||
|
tags:
|
||||||
|
- vmware/cascade-kcl-toolchain:latest
|
||||||
|
|
||||||
|
push-toolchain:
|
||||||
|
description: push toolchain image
|
||||||
|
after:
|
||||||
|
- rebuild-toolchain
|
||||||
|
push:
|
||||||
|
- vmware/cascade-toolchain:latest
|
||||||
|
|
||||||
|
settings:
|
||||||
|
default-targets:
|
||||||
|
- rebuild-toolchain
|
||||||
|
docker:
|
||||||
|
image: 'vmware/cascade-kcl-toolchain:0.0.0'
|
||||||
10
support/toolchain/docker/Dockerfile
Normal file
10
support/toolchain/docker/Dockerfile
Normal file
|
|
@ -0,0 +1,10 @@
|
||||||
|
FROM golang:1.10
|
||||||
|
ENV PATH /go/bin:/src/bin:/root/go/bin:/usr/local/go/bin:$PATH
|
||||||
|
ENV GOPATH /go:/src
|
||||||
|
RUN go get -v github.com/alecthomas/gometalinter && \
|
||||||
|
go get -v golang.org/x/tools/cmd/... && \
|
||||||
|
go get -v github.com/FiloSottile/gvt && \
|
||||||
|
go get -v github.com/GoASTScanner/gas/cmd/gas/... && \
|
||||||
|
go get github.com/derekparker/delve/cmd/dlv && \
|
||||||
|
gometalinter --install && \
|
||||||
|
chmod -R a+rw /go
|
||||||
Loading…
Reference in a new issue