From 22de13ef8a15dcad767525bc996995ca57796896 Mon Sep 17 00:00:00 2001 From: Tao Jiang Date: Mon, 27 Aug 2018 12:23:20 -0700 Subject: [PATCH] Go-KCL: Update security scan gas is now gosec. Need to update security scan and fix security issue as needed. No functional change. Jira CNA-2022 Change-Id: I36f2a204114f3f13e2ed05579c04a9c89f528f9a --- HyperMake | 3 ++- clientlibrary/metrics/cloudwatch.go | 7 ++++--- clientlibrary/worker/shard-consumer.go | 2 +- clientlibrary/worker/worker.go | 15 ++++++++++++--- support/scripts/check.sh | 2 +- support/toolchain/HyperMake | 10 +++++----- support/toolchain/docker/Dockerfile | 2 +- 7 files changed, 26 insertions(+), 15 deletions(-) diff --git a/HyperMake b/HyperMake index 03531b2..e006d3b 100644 --- a/HyperMake +++ b/HyperMake @@ -80,6 +80,7 @@ targets: after: - checkfmt - lint + - scanast deps-kcl: description: populate vendor packages @@ -95,5 +96,5 @@ settings: default-targets: - ci docker: - image: 'vmware/cascade-toolchain:0.0.9' + image: 'vmware/go-kcl-toolchain:latest' src-volume: /go/src/vmware.com/cascade-kinesis-client diff --git a/clientlibrary/metrics/cloudwatch.go b/clientlibrary/metrics/cloudwatch.go index 5da25bb..1a157f4 100644 --- a/clientlibrary/metrics/cloudwatch.go +++ b/clientlibrary/metrics/cloudwatch.go @@ -97,15 +97,16 @@ func (cw *CloudWatchMonitoringService) eventloop() { defer cw.waitGroup.Done() for { - err := cw.flush() - if err != nil { + if err := cw.flush(); err != nil { log.Errorf("Error sending metrics to CloudWatch. %+v", err) } select { case <-*cw.stop: log.Info("Shutting down monitoring system") - cw.flush() + if err := cw.flush(); err != nil { + log.Errorf("Error sending metrics to CloudWatch. %+v", err) + } return case <-time.After(time.Duration(cw.MetricsBufferTimeMillis) * time.Millisecond): } diff --git a/clientlibrary/worker/shard-consumer.go b/clientlibrary/worker/shard-consumer.go index 5666da8..9b60609 100644 --- a/clientlibrary/worker/shard-consumer.go +++ b/clientlibrary/worker/shard-consumer.go @@ -181,7 +181,7 @@ func (sc *ShardConsumer) getRecords(shard *shardStatus) error { continue } } - log.Errorf("Error getting records from Kinesis that cannot be retried: %+v\nRequest: %s", err, getRecordsArgs) + log.Errorf("Error getting records from Kinesis that cannot be retried: %+v Request: %s", err, getRecordsArgs) return err } retriedErrors = 0 diff --git a/clientlibrary/worker/worker.go b/clientlibrary/worker/worker.go index 0ab2a97..f430b8d 100644 --- a/clientlibrary/worker/worker.go +++ b/clientlibrary/worker/worker.go @@ -135,7 +135,10 @@ func (w *Worker) Start() error { // Start monitoring service log.Info("Starting monitoring service.") - w.mService.Start() + if err := w.mService.Start(); err != nil { + log.Errorf("Failed to start monitoring service: %+v", err) + return err + } log.Info("Starting worker event loop.") // entering event loop @@ -249,7 +252,7 @@ func (w *Worker) eventLoop() { if err != nil { // checkpoint may not existed yet is not an error condition. if err != ErrSequenceIDNotFound { - log.Error(err) + log.Errorf(" Error: %+v", err) // move on to next shard continue } @@ -308,10 +311,12 @@ func (w *Worker) getShardIDs(startShardID string, shardInfo map[string]bool) err streamDesc, err := w.kc.DescribeStream(args) if err != nil { + log.Errorf("Error in DescribeStream: %s Error: %+v Request: %s", w.streamName, err, args) return err } if *streamDesc.StreamDescription.StreamStatus != "ACTIVE" { + log.Warnf("Stream %s is not active", w.streamName) return errors.New("stream not active") } @@ -319,6 +324,7 @@ func (w *Worker) getShardIDs(startShardID string, shardInfo map[string]bool) err for _, s := range streamDesc.StreamDescription.Shards { // record avail shardId from fresh reading from Kinesis shardInfo[*s.ShardId] = true + // found new shard if _, ok := w.shardStatus[*s.ShardId]; !ok { log.Debugf("Found shard with id %s", *s.ShardId) @@ -336,6 +342,7 @@ func (w *Worker) getShardIDs(startShardID string, shardInfo map[string]bool) err if *streamDesc.StreamDescription.HasMoreShards { err := w.getShardIDs(lastShardID, shardInfo) if err != nil { + log.Errorf("Error in getShardIDs: %s Error: %+v", lastShardID, err) return err } } @@ -359,7 +366,9 @@ func (w *Worker) syncShard() error { delete(w.shardStatus, shard.ID) // remove the shard entry in dynamoDB as well // Note: syncShard runs periodically. we don't need to do anything in case of error here. - w.checkpointer.RemoveLeaseInfo(shard.ID) + if err := w.checkpointer.RemoveLeaseInfo(shard.ID); err != nil { + log.Errorf("Failed to remove shard lease info: %s Error: %+v", shard.ID, err) + } } } diff --git a/support/scripts/check.sh b/support/scripts/check.sh index 539bdf9..6126dc0 100755 --- a/support/scripts/check.sh +++ b/support/scripts/check.sh @@ -27,7 +27,7 @@ lint() { scanast() { set +e - gas ./... > security.log 2>&1 + gosec ./... > security.log 2>&1 set -e local issues=$(grep -E "Severity: MEDIUM" security.log | wc -l) diff --git a/support/toolchain/HyperMake b/support/toolchain/HyperMake index 708ccf3..40e36e7 100644 --- a/support/toolchain/HyperMake +++ b/support/toolchain/HyperMake @@ -1,8 +1,8 @@ --- format: hypermake.v0 -name: cascade-kcl -description: Amazon Kinesis Client Library in Go +name: go-kcl +description: VMWare Go-KCL Amazon Kinesis Client Library in Go targets: rebuild-toolchain: @@ -12,17 +12,17 @@ targets: build: docker cache: false tags: - - vmware/cascade-kcl-toolchain:latest + - vmware/go-kcl-toolchain:latest push-toolchain: description: push toolchain image after: - rebuild-toolchain push: - - vmware/cascade-toolchain:latest + - vmware/go-kcl-toolchain:latest settings: default-targets: - rebuild-toolchain docker: - image: 'vmware/cascade-kcl-toolchain:0.0.0' + image: 'vmware/go-kcl-toolchain:0.1.0' diff --git a/support/toolchain/docker/Dockerfile b/support/toolchain/docker/Dockerfile index 764f92f..5c36811 100644 --- a/support/toolchain/docker/Dockerfile +++ b/support/toolchain/docker/Dockerfile @@ -4,7 +4,7 @@ ENV GOPATH /go:/src RUN go get -v github.com/alecthomas/gometalinter && \ go get -v golang.org/x/tools/cmd/... && \ go get -v github.com/FiloSottile/gvt && \ - go get -v github.com/GoASTScanner/gas/cmd/gas/... && \ + go get github.com/securego/gosec/cmd/gosec/... && \ go get github.com/derekparker/delve/cmd/dlv && \ gometalinter --install && \ chmod -R a+rw /go \ No newline at end of file