a323d2fd51
This is the core part of KCL by implementing worker. It has exactly the same interface as Amazon's KCL. Internally, it uses code from GoKini in order to get the library functionaly quickly. This is a working version. The test code worker_test.go shows how to use this library. Dynamic resharding feature is out of the scope of M4. Test: 1. A Kinesis stream named "kcl-test" has been created under photon-infra account. 2. Download your AWS Credential from IAM user page. 3. Modify the worker_test.go to fill in your aws credential. 4. hmake test Jira CNA-637 Change-Id: I886d255bab9adaf7a13bca11bfda51bedaacaaed
67 lines
1.7 KiB
Bash
67 lines
1.7 KiB
Bash
set -ex
|
|
|
|
# PROJ_ROOT specifies the project root
|
|
export PROJ_ROOT="$HMAKE_PROJECT_DIR"
|
|
|
|
# Add /go in GOPATH because that's the original GOPATH in toolchain
|
|
export GOPATH=/go:$PROJ_ROOT
|
|
|
|
local_go_pkgs() {
|
|
find . -name '*.go' | \
|
|
grep -Fv '/vendor/' | \
|
|
grep -Fv '/go/' | \
|
|
grep -Fv '/gen/' | \
|
|
grep -Fv '/tmp/' | \
|
|
grep -Fv '/run/' | \
|
|
grep -Fv '/tests/' | \
|
|
grep -Fv '/gokini/' | \
|
|
sed -r 's|(.+)/[^/]+\.go$|\1|g' | \
|
|
sort -u
|
|
}
|
|
|
|
local_test_pkgs() {
|
|
find ./src/test -name '*.go' | \
|
|
grep -Fv '_test.go' | \
|
|
sed -r 's|(.+)/[^/]+\.go$|\1|g' | \
|
|
sort -u
|
|
}
|
|
|
|
version_suffix() {
|
|
local suffix=$(git log -1 --format=%h 2>/dev/null || true)
|
|
if [ -n "$suffix" ]; then
|
|
test -z "$(git status --porcelain 2>/dev/null || true)" || suffix="${suffix}+"
|
|
echo -n "-g${suffix}"
|
|
else
|
|
echo -n -dev
|
|
fi
|
|
}
|
|
|
|
git_commit_hash() {
|
|
echo $(git rev-parse --short HEAD)
|
|
}
|
|
|
|
# Due to Go plugin genhash algorithm simply takes full source path
|
|
# from archive, it generates different plugin hash if source path of
|
|
# shared pkg is different, and causes load failure.
|
|
# as a workaround, lookup shared pkg and place it to fixed path
|
|
FIX_GOPATH=/tmp/go
|
|
|
|
fix_go_pkg() {
|
|
local pkg="$1" base
|
|
for p in ${GOPATH//:/ }; do
|
|
if [ -d "$p/src/$pkg" ]; then
|
|
base="$p"
|
|
break
|
|
fi
|
|
done
|
|
|
|
if [ -z "$base" ]; then
|
|
echo "Package $pkg not found in GOPATH: $GOPATH" >&2
|
|
return 1
|
|
fi
|
|
|
|
local fix_pkg_path="$FIX_GOPATH/src/$pkg"
|
|
rm -f "$fix_pkg_path"
|
|
mkdir -p "$(dirname $fix_pkg_path)"
|
|
ln -s "$base/src/$pkg" "$fix_pkg_path"
|
|
}
|