CKL: Initial version for starting repository
Change-Id: Ic8dc81a437d9c72e841048359745e245c6c4895b
This commit is contained in:
Tao Jiang
parent
f7763e6d8b
commit
7b35571d9e
11 changed files with 481 additions and 0 deletions
18
.gitignore
vendored
18
.gitignore
vendored
|
|
@ -2,3 +2,21 @@
|
|||
*
|
||||
# Except this file
|
||||
!.gitignore
|
||||
=======
|
||||
/src/gen
|
||||
/src/vendor
|
||||
!/src/vendor/manifest
|
||||
/bin
|
||||
/pkg
|
||||
/tmp
|
||||
/log
|
||||
/vms
|
||||
/run
|
||||
/go
|
||||
.hmake
|
||||
.hmakerc
|
||||
.project
|
||||
.idea
|
||||
.vscode
|
||||
*_mock_test.go
|
||||
filenames
|
||||
|
|
|
|||
5
.gitreview
Normal file
5
.gitreview
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
[gerrit]
|
||||
host=review.ec.eng.vmware.com
|
||||
port=29418
|
||||
project=cascade-kinesis-client
|
||||
defaultbranch=develop
|
||||
99
HyperMake
Normal file
99
HyperMake
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
---
|
||||
format: hypermake.v0
|
||||
|
||||
name: cascade-kinesis-client
|
||||
description: Kinesis Client in Go
|
||||
|
||||
targets:
|
||||
rebuild-toolchain:
|
||||
description: build toolchain image
|
||||
watches:
|
||||
- support/docker/toolchain
|
||||
build: support/docker/toolchain
|
||||
|
||||
toolchain:
|
||||
description: placeholder for additional toolchain dependencies
|
||||
|
||||
deps:
|
||||
description: setup dependencies
|
||||
after:
|
||||
- 'deps-*'
|
||||
|
||||
gen:
|
||||
description: generate source code
|
||||
after:
|
||||
- 'gen-*'
|
||||
|
||||
build:
|
||||
description: build source code
|
||||
after:
|
||||
- 'build-*'
|
||||
|
||||
test:
|
||||
description: run unit tests
|
||||
after:
|
||||
- deps
|
||||
- gen
|
||||
always: true
|
||||
cmds:
|
||||
- ./support/scripts/test.sh
|
||||
|
||||
ci:
|
||||
description: run CI tests
|
||||
after:
|
||||
- check
|
||||
cmds:
|
||||
- ./support/scripts/ci.sh
|
||||
|
||||
checkfmt:
|
||||
description: check code format
|
||||
after:
|
||||
- toolchain
|
||||
watches:
|
||||
- support/scripts/check.sh
|
||||
always: true
|
||||
cmds:
|
||||
- ./support/scripts/check.sh fmt
|
||||
|
||||
lint:
|
||||
description: run lint to check code
|
||||
after:
|
||||
- toolchain
|
||||
watches:
|
||||
- support/scripts/check.sh
|
||||
always: true
|
||||
cmds:
|
||||
- ./support/scripts/check.sh lint
|
||||
|
||||
scanast:
|
||||
description: run Go AST security scan
|
||||
after:
|
||||
- toolchain
|
||||
watches:
|
||||
- '**/**/*.go'
|
||||
- './support/scripts/check.sh'
|
||||
cmds:
|
||||
- ./support/scripts/check.sh scanast
|
||||
|
||||
check:
|
||||
description: run all code checks
|
||||
after:
|
||||
- checkfmt
|
||||
- lint
|
||||
|
||||
deps-kcl:
|
||||
description: populate vendor packages
|
||||
after:
|
||||
- toolchain
|
||||
watches:
|
||||
- src/vendor/manifest
|
||||
workdir: src
|
||||
cmds:
|
||||
- gvt restore
|
||||
|
||||
settings:
|
||||
default-targets:
|
||||
- ci
|
||||
docker:
|
||||
image: 'vmware/cascade-kcl-toolchain:latest'
|
||||
src-volume: /home/cascade-kinesis-client
|
||||
|
|
@ -27,3 +27,9 @@ as an open-source patch. For more detailed information, refer to [CONTRIBUTING.m
|
|||
|
||||
## License
|
||||
|
||||
=======
|
||||
# Cascade Kinesis Client Library for GO
|
||||
|
||||
The **Cascade Kinesis Client Library for GO** (Cascade KCL) enables Go developers to easily consume and process data from [Amazon Kinesis][kinesis].
|
||||
|
||||
It is a re-implementation on Amazon's Kinesis Client Library in pure Go without using KCL's multi-language support.
|
||||
|
|
|
|||
167
src/vendor/manifest
vendored
Normal file
167
src/vendor/manifest
vendored
Normal file
|
|
@ -0,0 +1,167 @@
|
|||
{
|
||||
"version": 0,
|
||||
"dependencies": [
|
||||
{
|
||||
"importpath": "github.com/aws/aws-sdk-go",
|
||||
"repository": "https://github.com/aws/aws-sdk-go",
|
||||
"vcs": "git",
|
||||
"revision": "365b4d34369496e650e3056b33fce4e1a25cfc72",
|
||||
"branch": "master",
|
||||
"notests": true
|
||||
},
|
||||
{
|
||||
"importpath": "github.com/asaskevich/govalidator",
|
||||
"repository": "https://github.com/asaskevich/govalidator",
|
||||
"vcs": "git",
|
||||
"revision": "38ddb4612a5dfc2878731749ee825853d9f0aaa1",
|
||||
"branch": "master",
|
||||
"notests": true
|
||||
},
|
||||
{
|
||||
"importpath": "github.com/sirupsen/logrus",
|
||||
"repository": "https://github.com/sirupsen/logrus",
|
||||
"vcs": "git",
|
||||
"revision": "51dc0fc64317a2861273909081f9c315786533eb",
|
||||
"branch": "master",
|
||||
"notests": true
|
||||
},
|
||||
{
|
||||
"importpath": "github.com/beorn7/perks/quantile",
|
||||
"repository": "https://github.com/beorn7/perks",
|
||||
"vcs": "git",
|
||||
"revision": "4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9",
|
||||
"branch": "master",
|
||||
"path": "/quantile",
|
||||
"notests": true
|
||||
},
|
||||
{
|
||||
"importpath": "gopkg.in/gemnasium/logrus-airbrake-hook.v2",
|
||||
"repository": "https://gopkg.in/gemnasium/logrus-airbrake-hook.v2",
|
||||
"vcs": "git",
|
||||
"revision": "e928b033a891c0175fb643d5aa0779e86325eb12",
|
||||
"branch": "master",
|
||||
"notests": true
|
||||
},
|
||||
{
|
||||
"importpath": "github.com/matttproud/golang_protobuf_extensions/pbutil",
|
||||
"repository": "https://github.com/matttproud/golang_protobuf_extensions",
|
||||
"vcs": "git",
|
||||
"revision": "c12348ce28de40eed0136aa2b644d0ee0650e56c",
|
||||
"branch": "master",
|
||||
"path": "/pbutil",
|
||||
"notests": true
|
||||
},
|
||||
{
|
||||
"importpath": "github.com/prometheus/client_golang/prometheus",
|
||||
"repository": "https://github.com/prometheus/client_golang",
|
||||
"vcs": "git",
|
||||
"revision": "26b897001974f2b4ee6688377873e4d6f61d533c",
|
||||
"branch": "master",
|
||||
"path": "prometheus",
|
||||
"notests": true
|
||||
},
|
||||
{
|
||||
"importpath": "github.com/prometheus/client_model/go",
|
||||
"repository": "https://github.com/prometheus/client_model",
|
||||
"vcs": "git",
|
||||
"revision": "6f3806018612930941127f2a7c6c453ba2c527d2",
|
||||
"branch": "master",
|
||||
"path": "/go",
|
||||
"notests": true
|
||||
},
|
||||
{
|
||||
"importpath": "github.com/prometheus/common/expfmt",
|
||||
"repository": "https://github.com/prometheus/common",
|
||||
"vcs": "git",
|
||||
"revision": "3e6a7635bac6573d43f49f97b47eb9bda195dba8",
|
||||
"branch": "master",
|
||||
"path": "/expfmt",
|
||||
"notests": true
|
||||
},
|
||||
{
|
||||
"importpath": "github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg",
|
||||
"repository": "https://github.com/prometheus/common",
|
||||
"vcs": "git",
|
||||
"revision": "3e6a7635bac6573d43f49f97b47eb9bda195dba8",
|
||||
"branch": "master",
|
||||
"path": "internal/bitbucket.org/ww/goautoneg",
|
||||
"notests": true
|
||||
},
|
||||
{
|
||||
"importpath": "github.com/prometheus/common/model",
|
||||
"repository": "https://github.com/prometheus/common",
|
||||
"vcs": "git",
|
||||
"revision": "3e6a7635bac6573d43f49f97b47eb9bda195dba8",
|
||||
"branch": "master",
|
||||
"path": "model",
|
||||
"notests": true
|
||||
},
|
||||
{
|
||||
"importpath": "github.com/astaxie/beego/cache",
|
||||
"repository": "https://github.com/astaxie/beego",
|
||||
"vcs": "git",
|
||||
"revision": "a7354d2d084003e4122d6e69f7e5ab594fd117b2",
|
||||
"branch": "master",
|
||||
"path": "cache",
|
||||
"notests": true
|
||||
},
|
||||
{
|
||||
"importpath": "github.com/prometheus/procfs",
|
||||
"repository": "https://github.com/prometheus/procfs",
|
||||
"vcs": "git",
|
||||
"revision": "e645f4e5aaa8506fc71d6edbc5c4ff02c04c46f2",
|
||||
"branch": "master",
|
||||
"notests": true
|
||||
},
|
||||
{
|
||||
"importpath": "github.com/golang/protobuf/proto",
|
||||
"repository": "https://github.com/golang/protobuf",
|
||||
"vcs": "git",
|
||||
"revision": "6a1fa9404c0aebf36c879bc50152edcc953910d2",
|
||||
"branch": "master",
|
||||
"path": "/proto",
|
||||
"notests": true
|
||||
},
|
||||
{
|
||||
"importpath": "github.com/golang/protobuf/ptypes/any",
|
||||
"repository": "https://github.com/golang/protobuf",
|
||||
"vcs": "git",
|
||||
"revision": "6a1fa9404c0aebf36c879bc50152edcc953910d2",
|
||||
"branch": "master",
|
||||
"path": "ptypes/any",
|
||||
"notests": true
|
||||
},
|
||||
{
|
||||
"importpath": "github.com/google/uuid",
|
||||
"repository": "https://github.com/google/uuid",
|
||||
"vcs": "git",
|
||||
"revision": "6a5e28554805e78ea6141142aba763936c4761c0",
|
||||
"branch": "master",
|
||||
"notests": true
|
||||
},
|
||||
{
|
||||
"importpath": "github.com/matryer/try",
|
||||
"repository": "https://github.com/matryer/try",
|
||||
"vcs": "git",
|
||||
"revision": "312d2599e12e89ca89b52a09597394f449235d80",
|
||||
"branch": "master",
|
||||
"notests": true
|
||||
},
|
||||
{
|
||||
"importpath": "github.com/stretchr/testify",
|
||||
"repository": "https://github.com/stretchr/testify",
|
||||
"vcs": "git",
|
||||
"revision": "12b6f73e6084dad08a7c6e575284b177ecafbc71",
|
||||
"branch": "master",
|
||||
"notests": true
|
||||
},
|
||||
{
|
||||
"importpath": "gopkg.in/yaml.v2",
|
||||
"repository": "https://gopkg.in/yaml.v2",
|
||||
"vcs": "git",
|
||||
"revision": "cd8b52f8269e0feb286dfeef29f8fe4d5b397e0b",
|
||||
"branch": "v2",
|
||||
"notests": true
|
||||
}
|
||||
]
|
||||
}
|
||||
76
support/scripts/check.sh
Executable file
76
support/scripts/check.sh
Executable file
|
|
@ -0,0 +1,76 @@
|
|||
#!/bin/bash
|
||||
|
||||
. support/scripts/functions.sh
|
||||
|
||||
checkfmt() {
|
||||
local files="$(gofmt -l $(local_go_pkgs))"
|
||||
if [ -n "$files" ]; then
|
||||
echo "You need to run \"gofmt -w ./\" to fix your formating."
|
||||
echo "$files" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
lint() {
|
||||
gometalinter \
|
||||
--exclude=_mock.go \
|
||||
--disable=gotype \
|
||||
--vendor \
|
||||
--skip=test \
|
||||
--fast \
|
||||
--deadline=600s \
|
||||
--severity=golint:error \
|
||||
--errors \
|
||||
$(local_go_pkgs)
|
||||
}
|
||||
|
||||
scanast() {
|
||||
set +e
|
||||
gas ./... > security.log 2>&1
|
||||
set -e
|
||||
|
||||
local issues=$(grep -E "Severity: MEDIUM" security.log | wc -l)
|
||||
if [ -n $issues ] && [ $issues -gt 0 ]; then
|
||||
echo ""
|
||||
echo "Medium Severity Issues:"
|
||||
grep -E "Severity: MEDIUM" -A 1 security.log
|
||||
echo $issues "medium severity issues found."
|
||||
fi
|
||||
|
||||
local issues=$(grep -E "Severity: HIGH" security.log | grep -v "vendor")
|
||||
local issues_count=$(grep -E "Severity: HIGH" security.log | grep -v "vendor" | wc -l)
|
||||
if [ -n $issues_count ] && [ $issues_count -gt 0 ]; then
|
||||
echo ""
|
||||
echo "High Severity Issues:"
|
||||
grep -E "Severity: HIGH" -A 1 security.log
|
||||
echo $issues_count "high severity issues found."
|
||||
echo $issues
|
||||
echo "You need to resolve the high severity issues at the least."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
local issues=$(grep -E "Errors unhandled" security.log | grep -v "vendor" | grep -v "/src/go/src")
|
||||
local issues_count=$(grep -E "Errors unhandled" security.log | grep -v "vendor" | grep -v "/src/go/src" | wc -l)
|
||||
if [ -n $issues_count ] && [ $issues_count -gt 0 ]; then
|
||||
echo ""
|
||||
echo "Unhandled errors:"
|
||||
grep -E "Errors unhandled" security.log
|
||||
echo $issues_count "unhandled errors, please indicate with the right comment that this case is ok, or handle the error."
|
||||
echo $issues
|
||||
echo "You need to resolve the all unhandled errors."
|
||||
exit 1
|
||||
fi
|
||||
rm security.log
|
||||
}
|
||||
|
||||
usage() {
|
||||
echo "check.sh fmt|lint" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
case "$1" in
|
||||
fmt) checkfmt ;;
|
||||
lint) lint ;;
|
||||
scanast) scanast;;
|
||||
*) usage ;;
|
||||
esac
|
||||
2
support/scripts/ci.sh
Executable file
2
support/scripts/ci.sh
Executable file
|
|
@ -0,0 +1,2 @@
|
|||
#!/bin/bash
|
||||
./support/scripts/test.sh
|
||||
65
support/scripts/functions.sh
Normal file
65
support/scripts/functions.sh
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
set -ex
|
||||
|
||||
# PROJ_ROOT specifies the project root
|
||||
export PROJ_ROOT="$HMAKE_PROJECT_DIR"
|
||||
|
||||
# Add /go in GOPATH because that's the original GOPATH in toolchain
|
||||
export GOPATH=/go:$PROJ_ROOT
|
||||
|
||||
local_go_pkgs() {
|
||||
find . -name '*.go' | \
|
||||
grep -Fv '/vendor/' | \
|
||||
grep -Fv '/go/' | \
|
||||
grep -Fv '/gen/' | \
|
||||
grep -Fv '/tmp/' | \
|
||||
grep -Fv '/run/' | \
|
||||
grep -Fv '/tests/' | \
|
||||
sort -u
|
||||
}
|
||||
|
||||
local_test_pkgs() {
|
||||
find ./src/test -name '*.go' | \
|
||||
grep -Fv '_test.go' | \
|
||||
sed -r 's|(.+)/[^/]+\.go$|\1|g' | \
|
||||
sort -u
|
||||
}
|
||||
|
||||
version_suffix() {
|
||||
local suffix=$(git log -1 --format=%h 2>/dev/null || true)
|
||||
if [ -n "$suffix" ]; then
|
||||
test -z "$(git status --porcelain 2>/dev/null || true)" || suffix="${suffix}+"
|
||||
echo -n "-g${suffix}"
|
||||
else
|
||||
echo -n -dev
|
||||
fi
|
||||
}
|
||||
|
||||
git_commit_hash() {
|
||||
echo $(git rev-parse --short HEAD)
|
||||
}
|
||||
|
||||
# Due to Go plugin genhash algorithm simply takes full source path
|
||||
# from archive, it generates different plugin hash if source path of
|
||||
# shared pkg is different, and causes load failure.
|
||||
# as a workaround, lookup shared pkg and place it to fixed path
|
||||
FIX_GOPATH=/tmp/go
|
||||
|
||||
fix_go_pkg() {
|
||||
local pkg="$1" base
|
||||
for p in ${GOPATH//:/ }; do
|
||||
if [ -d "$p/src/$pkg" ]; then
|
||||
base="$p"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -z "$base" ]; then
|
||||
echo "Package $pkg not found in GOPATH: $GOPATH" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
local fix_pkg_path="$FIX_GOPATH/src/$pkg"
|
||||
rm -f "$fix_pkg_path"
|
||||
mkdir -p "$(dirname $fix_pkg_path)"
|
||||
ln -s "$base/src/$pkg" "$fix_pkg_path"
|
||||
}
|
||||
5
support/scripts/test.sh
Executable file
5
support/scripts/test.sh
Executable file
|
|
@ -0,0 +1,5 @@
|
|||
#!/bin/bash
|
||||
. support/scripts/functions.sh
|
||||
|
||||
# Run only the unit tests and not integration tests
|
||||
go test -race $(local_go_pkgs)
|
||||
28
support/toolchain/HyperMake
Normal file
28
support/toolchain/HyperMake
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
---
|
||||
format: hypermake.v0
|
||||
|
||||
name: cascade-kcl
|
||||
description: Amazon Kinesis Client Library in Go
|
||||
|
||||
targets:
|
||||
rebuild-toolchain:
|
||||
description: build toolchain image
|
||||
watches:
|
||||
- docker
|
||||
build: docker
|
||||
cache: false
|
||||
tags:
|
||||
- vmware/cascade-kcl-toolchain:latest
|
||||
|
||||
push-toolchain:
|
||||
description: push toolchain image
|
||||
after:
|
||||
- rebuild-toolchain
|
||||
push:
|
||||
- vmware/cascade-toolchain:latest
|
||||
|
||||
settings:
|
||||
default-targets:
|
||||
- rebuild-toolchain
|
||||
docker:
|
||||
image: 'vmware/cascade-kcl-toolchain:0.0.0'
|
||||
10
support/toolchain/docker/Dockerfile
Normal file
10
support/toolchain/docker/Dockerfile
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
FROM golang:1.10
|
||||
ENV PATH /go/bin:/src/bin:/root/go/bin:/usr/local/go/bin:$PATH
|
||||
ENV GOPATH /go:/src
|
||||
RUN go get -v github.com/alecthomas/gometalinter && \
|
||||
go get -v golang.org/x/tools/cmd/... && \
|
||||
go get -v github.com/FiloSottile/gvt && \
|
||||
go get -v github.com/GoASTScanner/gas/cmd/gas/... && \
|
||||
go get github.com/derekparker/delve/cmd/dlv && \
|
||||
gometalinter --install && \
|
||||
chmod -R a+rw /go
|
||||
Loading…
Reference in a new issue