luciano/amazon-kinesis-client-go
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

splitter: don't filter by env

Browse source 
Filtering by env was an artifact of the previous way we originally had
setup our CWLogs subscriptions, assuming that we could have multiple
subscriptions per stream. :( You can't.

Let's instead allow any CWLogs subscription's logs to be valid, which is
needed for:
https://github.com/Clever/kinesis-cwlogs-splitter/pull/1
This commit is contained in:
Nathan Leiby 2017-08-10 14:22:58 -07:00
parent 16814bd597
commit 0c02e93821
2 changed files with 4 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
splitter/splitter.go
View file

@ -34,12 +34,12 @@ func IsGzipped(b []byte) bool {
// GetMessagesFromGzippedInput takes a gzipped record from a CWLogs Subscription and splits it into
// a slice of messages.
func GetMessagesFromGzippedInput(input []byte, prodEnv bool) ([][]byte, error) {
func GetMessagesFromGzippedInput(input []byte) ([][]byte, error) {
unpacked, err := unpack(input)
if err != nil {
return [][]byte{}, err
}
return Split(unpacked, prodEnv), nil
return Split(unpacked), nil
}
// Unpack expects a gzipped + json-stringified LogEventBatch
@ -74,7 +74,7 @@ var taskRegex = regexp.MustCompile(taskMeta)
// Split takes a LogEventBatch and separates into a slice of enriched log lines
// Lines are enhanced by adding an Rsyslog prefix, which should be handled correctly by
// the subsequent decoding logic.
func Split(b LogEventBatch, prodEnv bool) [][]byte {
func Split(b LogEventBatch) [][]byte {
env := "unknown"
app := "unknown"
task := "00001111-2222-3333-4444-555566667777"
@ -85,12 +85,6 @@ func Split(b LogEventBatch, prodEnv bool) [][]byte {
task = matches[0][3]
}
if (env == "production") != prodEnv {
// if there's a mis-match between the consumer's environment and the log's environment,
// throw away the log. (this is a workaround for coarse grained subscription filters.)
return [][]byte{}
}
rsyslogPrefix := `%s %s %s[%d]: %s`
// programName is a mocked ARN in the format expected by our log decoders
programName := env + "--" + app + `/arn%3Aaws%3Aecs%3Aus-east-1%3A999988887777%3Atask%2F` + task

57
splitter/splitter_test.go
View file

@ -118,65 +118,10 @@ func TestSplit(t *testing.T) {
},
},
}
prodEnv := false
lines := Split(input, prodEnv)
lines := Split(input)
expected := [][]byte{
[]byte("2017-06-26T23:32:23.285001+00:00 aws-batch env--app/arn%3Aaws%3Aecs%3Aus-east-1%3A999988887777%3Atask%2F12345678-1234-1234-1234-555566667777[1]: some log line"),
[]byte("2017-06-26T23:32:23.285001+00:00 aws-batch env--app/arn%3Aaws%3Aecs%3Aus-east-1%3A999988887777%3Atask%2F12345678-1234-1234-1234-555566667777[1]: another log line"),
}
assert.Equal(t, expected, lines)
}
func TestSplitFiltersByEnv(t *testing.T) {
t.Log("If Split is run with prodEnv == true, it should omit logs with env != production")
input := LogEventBatch{
MessageType: "DATA_MESSAGE",
Owner: "123456789012",
LogGroup: "/aws/batch/job",
LogStream: "env--app/12345678-1234-1234-1234-555566667777/88889999-0000-aaaa-bbbb-ccccddddeeee",
// LogStream: "environment--app",
SubscriptionFilters: []string{"MySubscriptionFilter"},
LogEvents: []LogEvent{
{
ID: "99999992379011144044923130086453437181614530551221780480",
Timestamp: 1498519943285,
Message: "some log line",
},
{
ID: "99999992387663833181953011865369295871402094815542181889",
Timestamp: 1498519943285,
Message: "another log line",
},
},
}
prodEnv := true
lines := Split(input, prodEnv)
expected := [][]byte{}
assert.Equal(t, expected, lines)
t.Log("If Split is run with prodEnv == false, it should omit logs with env == production")
input = LogEventBatch{
MessageType: "DATA_MESSAGE",
Owner: "123456789012",
LogGroup: "/aws/batch/job",
LogStream: "production--app/12345678-1234-1234-1234-555566667777/88889999-0000-aaaa-bbbb-ccccddddeeee",
// LogStream: "environment--app",
SubscriptionFilters: []string{"MySubscriptionFilter"},
LogEvents: []LogEvent{
{
ID: "99999992379011144044923130086453437181614530551221780480",
Timestamp: 1498519943285,
Message: "some log line",
},
{
ID: "99999992387663833181953011865369295871402094815542181889",
Timestamp: 1498519943285,
Message: "another log line",
},
},
}
prodEnv = false
lines = Split(input, prodEnv)
expected = [][]byte{}
assert.Equal(t, expected, lines)
}