Merge pull request #19 from Clever/rm-params-from-decoder
Removed stringifyNested, renameESReservedFields, and minimumTimestamp params from decoder
This commit is contained in:
commit
94aacdf833
2 changed files with 37 additions and 196 deletions
|
|
@ -5,7 +5,6 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/Clever/syslogparser/rfc3164"
|
"github.com/Clever/syslogparser/rfc3164"
|
||||||
)
|
)
|
||||||
|
|
@ -174,6 +173,17 @@ func getStringArray(json map[string]interface{}, key string) []string {
|
||||||
// LogRoutes a type alias to make it easier to add route specific filter functions
|
// LogRoutes a type alias to make it easier to add route specific filter functions
|
||||||
type LogRoutes []map[string]interface{}
|
type LogRoutes []map[string]interface{}
|
||||||
|
|
||||||
|
// RuleNames returns a list of the names of the rules
|
||||||
|
func (l LogRoutes) RuleNames() []string {
|
||||||
|
names := []string{}
|
||||||
|
for _, route := range l {
|
||||||
|
name := getStringValue(route, "rule")
|
||||||
|
names = append(names, name)
|
||||||
|
}
|
||||||
|
|
||||||
|
return names
|
||||||
|
}
|
||||||
|
|
||||||
// MetricsRoutes filters the LogRoutes and returns a list of MetricsRoutes structs
|
// MetricsRoutes filters the LogRoutes and returns a list of MetricsRoutes structs
|
||||||
func (l LogRoutes) MetricsRoutes() []MetricsRoute {
|
func (l LogRoutes) MetricsRoutes() []MetricsRoute {
|
||||||
routes := []MetricsRoute{}
|
routes := []MetricsRoute{}
|
||||||
|
|
@ -354,7 +364,7 @@ func ExtractKVMeta(kvlog map[string]interface{}) KVMeta {
|
||||||
}
|
}
|
||||||
|
|
||||||
// ParseAndEnhance extracts fields from a log line, and does some post-processing to rename/add fields
|
// ParseAndEnhance extracts fields from a log line, and does some post-processing to rename/add fields
|
||||||
func ParseAndEnhance(line string, env string, stringifyNested bool, renameESReservedFields bool, minimumTimestamp time.Time) (map[string]interface{}, error) {
|
func ParseAndEnhance(line string, env string) (map[string]interface{}, error) {
|
||||||
out := map[string]interface{}{}
|
out := map[string]interface{}{}
|
||||||
|
|
||||||
syslogFields, err := FieldsFromSyslog(line)
|
syslogFields, err := FieldsFromSyslog(line)
|
||||||
|
|
@ -405,55 +415,9 @@ func ParseAndEnhance(line string, env string, stringifyNested bool, renameESRese
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ES dynamic mappings get finnicky once you start sending nested objects.
|
|
||||||
// E.g., if one app sends a field for the first time as an object, then any log
|
|
||||||
// sent by another app containing that field /not/ as an object will fail.
|
|
||||||
// One solution is to decode nested objects as strings.
|
|
||||||
if stringifyNested {
|
|
||||||
for k, v := range out {
|
|
||||||
_, ismap := v.(map[string]interface{})
|
|
||||||
_, isarr := v.([]interface{})
|
|
||||||
if ismap || isarr {
|
|
||||||
bs, _ := json.Marshal(v)
|
|
||||||
out[k] = string(bs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ES doesn't like fields that start with underscores.
|
|
||||||
if renameESReservedFields {
|
|
||||||
for oldKey, renamedKey := range esFieldRenames {
|
|
||||||
if val, ok := out[oldKey]; ok {
|
|
||||||
out[renamedKey] = val
|
|
||||||
delete(out, oldKey)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
msgTime, ok := out["timestamp"].(time.Time)
|
|
||||||
if ok && !msgTime.After(minimumTimestamp) {
|
|
||||||
return map[string]interface{}{}, fmt.Errorf("message's timestamp < minimumTimestamp")
|
|
||||||
}
|
|
||||||
|
|
||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
var esFieldRenames = map[string]string{
|
|
||||||
"_index": "kv__index",
|
|
||||||
"_uid": "kv__uid",
|
|
||||||
"_type": "kv__type",
|
|
||||||
"_id": "kv__id",
|
|
||||||
"_source": "kv__source",
|
|
||||||
"_size": "kv__size",
|
|
||||||
"_all": "kv__all",
|
|
||||||
"_field_names": "kv__field_names",
|
|
||||||
"_timestamp": "kv__timestamp",
|
|
||||||
"_ttl": "kv__ttl",
|
|
||||||
"_parent": "kv__parent",
|
|
||||||
"_routing": "kv__routing",
|
|
||||||
"_meta": "kv__meta",
|
|
||||||
}
|
|
||||||
|
|
||||||
const containerMeta = `([a-z0-9-]+)--([a-z0-9-]+)\/` + // env--app
|
const containerMeta = `([a-z0-9-]+)--([a-z0-9-]+)\/` + // env--app
|
||||||
`arn%3Aaws%3Aecs%3Aus-(west|east)-[1-2]%3A[0-9]{12}%3Atask%2F` + // ARN cruft
|
`arn%3Aaws%3Aecs%3Aus-(west|east)-[1-2]%3A[0-9]{12}%3Atask%2F` + // ARN cruft
|
||||||
`([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})` // task-id
|
`([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})` // task-id
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@ package decode
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"sort"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
|
@ -188,16 +189,9 @@ func TestSyslogDecoding(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type ParseAndEnhanceInput struct {
|
|
||||||
Line string
|
|
||||||
StringifyNested bool
|
|
||||||
RenameESReservedFields bool
|
|
||||||
MinimumTimestamp time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
type ParseAndEnhanceSpec struct {
|
type ParseAndEnhanceSpec struct {
|
||||||
Title string
|
Title string
|
||||||
Input ParseAndEnhanceInput
|
Line string
|
||||||
ExpectedOutput map[string]interface{}
|
ExpectedOutput map[string]interface{}
|
||||||
ExpectedError error
|
ExpectedError error
|
||||||
}
|
}
|
||||||
|
|
@ -219,7 +213,7 @@ func TestParseAndEnhance(t *testing.T) {
|
||||||
specs := []ParseAndEnhanceSpec{
|
specs := []ParseAndEnhanceSpec{
|
||||||
ParseAndEnhanceSpec{
|
ParseAndEnhanceSpec{
|
||||||
Title: "Parses a Kayvee log line from an ECS app",
|
Title: "Parses a Kayvee log line from an ECS app",
|
||||||
Input: ParseAndEnhanceInput{Line: `2017-04-05T21:57:46.794862+00:00 ip-10-0-0-0 env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef[3291]: 2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished"}`},
|
Line: `2017-04-05T21:57:46.794862+00:00 ip-10-0-0-0 env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef[3291]: 2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished"}`,
|
||||||
ExpectedOutput: map[string]interface{}{
|
ExpectedOutput: map[string]interface{}{
|
||||||
"timestamp": logTime3,
|
"timestamp": logTime3,
|
||||||
"hostname": "ip-10-0-0-0",
|
"hostname": "ip-10-0-0-0",
|
||||||
|
|
@ -238,7 +232,7 @@ func TestParseAndEnhance(t *testing.T) {
|
||||||
},
|
},
|
||||||
ParseAndEnhanceSpec{
|
ParseAndEnhanceSpec{
|
||||||
Title: "Parses a Kayvee log line from an ECS app, with override to container_app",
|
Title: "Parses a Kayvee log line from an ECS app, with override to container_app",
|
||||||
Input: ParseAndEnhanceInput{Line: `2017-04-05T21:57:46.794862+00:00 ip-10-0-0-0 env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef[3291]: 2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished","container_app":"force-app"}`},
|
Line: `2017-04-05T21:57:46.794862+00:00 ip-10-0-0-0 env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef[3291]: 2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished","container_app":"force-app"}`,
|
||||||
ExpectedOutput: map[string]interface{}{
|
ExpectedOutput: map[string]interface{}{
|
||||||
"timestamp": logTime3,
|
"timestamp": logTime3,
|
||||||
"hostname": "ip-10-0-0-0",
|
"hostname": "ip-10-0-0-0",
|
||||||
|
|
@ -257,7 +251,7 @@ func TestParseAndEnhance(t *testing.T) {
|
||||||
},
|
},
|
||||||
ParseAndEnhanceSpec{
|
ParseAndEnhanceSpec{
|
||||||
Title: "Parses a non-Kayvee log line",
|
Title: "Parses a non-Kayvee log line",
|
||||||
Input: ParseAndEnhanceInput{Line: `2017-04-05T21:57:46.794862+00:00 ip-10-0-0-0 env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef[3291]: some log`},
|
Line: `2017-04-05T21:57:46.794862+00:00 ip-10-0-0-0 env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef[3291]: some log`,
|
||||||
ExpectedOutput: map[string]interface{}{
|
ExpectedOutput: map[string]interface{}{
|
||||||
"timestamp": logTime3,
|
"timestamp": logTime3,
|
||||||
"hostname": "ip-10-0-0-0",
|
"hostname": "ip-10-0-0-0",
|
||||||
|
|
@ -272,13 +266,13 @@ func TestParseAndEnhance(t *testing.T) {
|
||||||
},
|
},
|
||||||
ParseAndEnhanceSpec{
|
ParseAndEnhanceSpec{
|
||||||
Title: "Fails to parse non-RSyslog log line",
|
Title: "Fails to parse non-RSyslog log line",
|
||||||
Input: ParseAndEnhanceInput{Line: `not rsyslog`},
|
Line: `not rsyslog`,
|
||||||
ExpectedOutput: map[string]interface{}{},
|
ExpectedOutput: map[string]interface{}{},
|
||||||
ExpectedError: &syslogparser.ParserError{},
|
ExpectedError: &syslogparser.ParserError{},
|
||||||
},
|
},
|
||||||
ParseAndEnhanceSpec{
|
ParseAndEnhanceSpec{
|
||||||
Title: "Parses JSON values",
|
Title: "Parses JSON values",
|
||||||
Input: ParseAndEnhanceInput{Line: `2017-04-05T21:57:46.794862+00:00 ip-10-0-0-0 env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef[3291]: 2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished", "nested": {"a":"b"}}`},
|
Line: `2017-04-05T21:57:46.794862+00:00 ip-10-0-0-0 env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef[3291]: 2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished", "nested": {"a":"b"}}`,
|
||||||
ExpectedOutput: map[string]interface{}{
|
ExpectedOutput: map[string]interface{}{
|
||||||
"timestamp": logTime3,
|
"timestamp": logTime3,
|
||||||
"hostname": "ip-10-0-0-0",
|
"hostname": "ip-10-0-0-0",
|
||||||
|
|
@ -296,141 +290,9 @@ func TestParseAndEnhance(t *testing.T) {
|
||||||
},
|
},
|
||||||
ExpectedError: nil,
|
ExpectedError: nil,
|
||||||
},
|
},
|
||||||
ParseAndEnhanceSpec{
|
|
||||||
Title: "Has the option to stringify object values",
|
|
||||||
Input: ParseAndEnhanceInput{
|
|
||||||
Line: `2017-04-05T21:57:46.794862+00:00 ip-10-0-0-0 env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef[3291]: 2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished", "nested": {"a":"b"}}`,
|
|
||||||
StringifyNested: true,
|
|
||||||
},
|
|
||||||
ExpectedOutput: map[string]interface{}{
|
|
||||||
"timestamp": logTime3,
|
|
||||||
"hostname": "ip-10-0-0-0",
|
|
||||||
"programname": `env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef`,
|
|
||||||
"rawlog": `2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished", "nested": {"a":"b"}}`,
|
|
||||||
"title": "request_finished",
|
|
||||||
"decoder_msg_type": "Kayvee",
|
|
||||||
"prefix": "2017/04/05 21:57:46 some_file.go:10: ",
|
|
||||||
"postfix": "",
|
|
||||||
"env": "deploy-env",
|
|
||||||
"container_env": "env",
|
|
||||||
"container_app": "app",
|
|
||||||
"container_task": "abcd1234-1a3b-1a3b-1234-d76552f4b7ef",
|
|
||||||
"nested": `{"a":"b"}`,
|
|
||||||
},
|
|
||||||
ExpectedError: nil,
|
|
||||||
},
|
|
||||||
ParseAndEnhanceSpec{
|
|
||||||
Title: "Has the option to stringify array values",
|
|
||||||
Input: ParseAndEnhanceInput{
|
|
||||||
Line: `2017-04-05T21:57:46.794862+00:00 ip-10-0-0-0 env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef[3291]: 2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished", "nested": [{"a":"b"}]}`,
|
|
||||||
StringifyNested: true,
|
|
||||||
},
|
|
||||||
ExpectedOutput: map[string]interface{}{
|
|
||||||
"timestamp": logTime3,
|
|
||||||
"hostname": "ip-10-0-0-0",
|
|
||||||
"programname": `env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef`,
|
|
||||||
"rawlog": `2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished", "nested": [{"a":"b"}]}`,
|
|
||||||
"title": "request_finished",
|
|
||||||
"decoder_msg_type": "Kayvee",
|
|
||||||
"prefix": "2017/04/05 21:57:46 some_file.go:10: ",
|
|
||||||
"postfix": "",
|
|
||||||
"env": "deploy-env",
|
|
||||||
"container_env": "env",
|
|
||||||
"container_app": "app",
|
|
||||||
"container_task": "abcd1234-1a3b-1a3b-1234-d76552f4b7ef",
|
|
||||||
"nested": `[{"a":"b"}]`,
|
|
||||||
},
|
|
||||||
ExpectedError: nil,
|
|
||||||
},
|
|
||||||
ParseAndEnhanceSpec{
|
|
||||||
Title: "Has the option to rename reserved ES fields",
|
|
||||||
Input: ParseAndEnhanceInput{
|
|
||||||
Line: `2017-04-05T21:57:46.794862+00:00 ip-10-0-0-0 env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef[3291]: 2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished", "_source": "a"}`,
|
|
||||||
RenameESReservedFields: true,
|
|
||||||
},
|
|
||||||
ExpectedOutput: map[string]interface{}{
|
|
||||||
"timestamp": logTime3,
|
|
||||||
"hostname": "ip-10-0-0-0",
|
|
||||||
"programname": `env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef`,
|
|
||||||
"rawlog": `2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished", "_source": "a"}`,
|
|
||||||
"title": "request_finished",
|
|
||||||
"decoder_msg_type": "Kayvee",
|
|
||||||
"prefix": "2017/04/05 21:57:46 some_file.go:10: ",
|
|
||||||
"postfix": "",
|
|
||||||
"env": "deploy-env",
|
|
||||||
"container_env": "env",
|
|
||||||
"container_app": "app",
|
|
||||||
"container_task": "abcd1234-1a3b-1a3b-1234-d76552f4b7ef",
|
|
||||||
"kv__source": "a",
|
|
||||||
},
|
|
||||||
ExpectedError: nil,
|
|
||||||
},
|
|
||||||
ParseAndEnhanceSpec{
|
|
||||||
Title: "Errors if logTime < MinimumTimestamp",
|
|
||||||
Input: ParseAndEnhanceInput{
|
|
||||||
Line: `2017-04-05T21:57:46.794862+00:00 ip-10-0-0-0 env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef[3291]: 2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished", "_source": "a"}`,
|
|
||||||
RenameESReservedFields: true,
|
|
||||||
MinimumTimestamp: time.Now().Add(100 * time.Hour * 24 * 365), // good thru year 2117
|
|
||||||
},
|
|
||||||
ExpectedOutput: map[string]interface{}{},
|
|
||||||
ExpectedError: fmt.Errorf(""),
|
|
||||||
},
|
|
||||||
ParseAndEnhanceSpec{
|
|
||||||
Title: "Accepts logs if logTime > MinimumTimestamp",
|
|
||||||
Input: ParseAndEnhanceInput{
|
|
||||||
Line: `2017-04-05T21:57:46.794862+00:00 ip-10-0-0-0 env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef[3291]: 2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished", "_source": "a"}`,
|
|
||||||
RenameESReservedFields: true,
|
|
||||||
MinimumTimestamp: time.Now().Add(-100 * time.Hour * 24 * 365), // good thru year 2117
|
|
||||||
},
|
|
||||||
ExpectedOutput: map[string]interface{}{
|
|
||||||
"timestamp": logTime3,
|
|
||||||
"hostname": "ip-10-0-0-0",
|
|
||||||
"programname": `env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef`,
|
|
||||||
"rawlog": `2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished", "_source": "a"}`,
|
|
||||||
"title": "request_finished",
|
|
||||||
"decoder_msg_type": "Kayvee",
|
|
||||||
"prefix": "2017/04/05 21:57:46 some_file.go:10: ",
|
|
||||||
"postfix": "",
|
|
||||||
"env": "deploy-env",
|
|
||||||
"container_env": "env",
|
|
||||||
"container_app": "app",
|
|
||||||
"container_task": "abcd1234-1a3b-1a3b-1234-d76552f4b7ef",
|
|
||||||
"kv__source": "a",
|
|
||||||
},
|
|
||||||
ExpectedError: nil,
|
|
||||||
},
|
|
||||||
ParseAndEnhanceSpec{
|
|
||||||
Title: "Accepts logs if logTime > MinimumTimestamp",
|
|
||||||
Input: ParseAndEnhanceInput{
|
|
||||||
Line: `2017-04-05T21:57:46.794862+00:00 ip-10-0-0-0 env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef[3291]: 2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished", "_source": "a"}`,
|
|
||||||
RenameESReservedFields: true,
|
|
||||||
MinimumTimestamp: time.Now().Add(-100 * time.Hour * 24 * 365), // good thru year 2117
|
|
||||||
},
|
|
||||||
ExpectedOutput: map[string]interface{}{
|
|
||||||
"timestamp": logTime3,
|
|
||||||
"hostname": "ip-10-0-0-0",
|
|
||||||
"programname": `env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef`,
|
|
||||||
"rawlog": `2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished", "_source": "a"}`,
|
|
||||||
"title": "request_finished",
|
|
||||||
"decoder_msg_type": "Kayvee",
|
|
||||||
"prefix": "2017/04/05 21:57:46 some_file.go:10: ",
|
|
||||||
"postfix": "",
|
|
||||||
"env": "deploy-env",
|
|
||||||
"container_env": "env",
|
|
||||||
"container_app": "app",
|
|
||||||
"container_task": "abcd1234-1a3b-1a3b-1234-d76552f4b7ef",
|
|
||||||
"kv__source": "a",
|
|
||||||
},
|
|
||||||
ExpectedError: nil,
|
|
||||||
},
|
|
||||||
ParseAndEnhanceSpec{
|
ParseAndEnhanceSpec{
|
||||||
Title: "Log with timestamp time.RFC3339 format",
|
Title: "Log with timestamp time.RFC3339 format",
|
||||||
Input: ParseAndEnhanceInput{
|
Line: `2017-04-05T21:57:46+00:00 mongo-docker-pipeline-r10-4 diamond[24099] Signal Received: 15`,
|
||||||
Line: `2017-04-05T21:57:46+00:00 mongo-docker-pipeline-r10-4 diamond[24099] ` +
|
|
||||||
`Signal Received: 15`,
|
|
||||||
RenameESReservedFields: true,
|
|
||||||
MinimumTimestamp: time.Now().Add(-100 * time.Hour * 24 * 365), // year 2117
|
|
||||||
},
|
|
||||||
ExpectedOutput: map[string]interface{}{
|
ExpectedOutput: map[string]interface{}{
|
||||||
"env": "deploy-env",
|
"env": "deploy-env",
|
||||||
"hostname": "mongo-docker-pipeline-r10-4",
|
"hostname": "mongo-docker-pipeline-r10-4",
|
||||||
|
|
@ -443,7 +305,7 @@ func TestParseAndEnhance(t *testing.T) {
|
||||||
for _, spec := range specs {
|
for _, spec := range specs {
|
||||||
t.Run(fmt.Sprintf(spec.Title), func(t *testing.T) {
|
t.Run(fmt.Sprintf(spec.Title), func(t *testing.T) {
|
||||||
assert := assert.New(t)
|
assert := assert.New(t)
|
||||||
fields, err := ParseAndEnhance(spec.Input.Line, "deploy-env", spec.Input.StringifyNested, spec.Input.RenameESReservedFields, spec.Input.MinimumTimestamp)
|
fields, err := ParseAndEnhance(spec.Line, "deploy-env")
|
||||||
if spec.ExpectedError != nil {
|
if spec.ExpectedError != nil {
|
||||||
assert.Error(err)
|
assert.Error(err)
|
||||||
assert.IsType(spec.ExpectedError, err)
|
assert.IsType(spec.ExpectedError, err)
|
||||||
|
|
@ -523,6 +385,7 @@ func TestExtractKVMeta(t *testing.T) {
|
||||||
Team string
|
Team string
|
||||||
Language string
|
Language string
|
||||||
Version string
|
Version string
|
||||||
|
Names []string
|
||||||
ExpectedMetricsRoutes []MetricsRoute
|
ExpectedMetricsRoutes []MetricsRoute
|
||||||
ExpectedAnalyticsRoutes []AnalyticsRoute
|
ExpectedAnalyticsRoutes []AnalyticsRoute
|
||||||
ExpectedNotificationRoutes []NotificationRoute
|
ExpectedNotificationRoutes []NotificationRoute
|
||||||
|
|
@ -530,10 +393,12 @@ func TestExtractKVMeta(t *testing.T) {
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
Description: "log line with no routes",
|
Description: "log line with no routes",
|
||||||
|
Names: []string{},
|
||||||
Log: map[string]interface{}{"hi": "hello!"},
|
Log: map[string]interface{}{"hi": "hello!"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
Description: "empty _kvmeta",
|
Description: "empty _kvmeta",
|
||||||
|
Names: []string{},
|
||||||
Log: map[string]interface{}{
|
Log: map[string]interface{}{
|
||||||
"hi": "hello!",
|
"hi": "hello!",
|
||||||
"_kvmeta": map[string]interface{}{},
|
"_kvmeta": map[string]interface{}{},
|
||||||
|
|
@ -544,6 +409,7 @@ func TestExtractKVMeta(t *testing.T) {
|
||||||
Team: "green",
|
Team: "green",
|
||||||
Version: "three",
|
Version: "three",
|
||||||
Language: "tree",
|
Language: "tree",
|
||||||
|
Names: []string{},
|
||||||
Log: map[string]interface{}{
|
Log: map[string]interface{}{
|
||||||
"hi": "hello!",
|
"hi": "hello!",
|
||||||
"_kvmeta": map[string]interface{}{
|
"_kvmeta": map[string]interface{}{
|
||||||
|
|
@ -558,6 +424,7 @@ func TestExtractKVMeta(t *testing.T) {
|
||||||
Team: "green",
|
Team: "green",
|
||||||
Version: "three",
|
Version: "three",
|
||||||
Language: "tree",
|
Language: "tree",
|
||||||
|
Names: []string{"cool", "cool2"},
|
||||||
ExpectedMetricsRoutes: []MetricsRoute{
|
ExpectedMetricsRoutes: []MetricsRoute{
|
||||||
{
|
{
|
||||||
Series: "1,1,2,3,5,8,13",
|
Series: "1,1,2,3,5,8,13",
|
||||||
|
|
@ -601,6 +468,7 @@ func TestExtractKVMeta(t *testing.T) {
|
||||||
Team: "green",
|
Team: "green",
|
||||||
Version: "christmas",
|
Version: "christmas",
|
||||||
Language: "tree",
|
Language: "tree",
|
||||||
|
Names: []string{"what's-this?", "there's-app-invites-everywhere"},
|
||||||
ExpectedAnalyticsRoutes: []AnalyticsRoute{
|
ExpectedAnalyticsRoutes: []AnalyticsRoute{
|
||||||
{
|
{
|
||||||
Series: "what-is-this",
|
Series: "what-is-this",
|
||||||
|
|
@ -637,6 +505,7 @@ func TestExtractKVMeta(t *testing.T) {
|
||||||
Team: "slack",
|
Team: "slack",
|
||||||
Version: "evergreen",
|
Version: "evergreen",
|
||||||
Language: "markdown-ish",
|
Language: "markdown-ish",
|
||||||
|
Names: []string{"did-you-know", "what's-the-catch"},
|
||||||
ExpectedNotificationRoutes: []NotificationRoute{
|
ExpectedNotificationRoutes: []NotificationRoute{
|
||||||
{
|
{
|
||||||
RuleName: "did-you-know",
|
RuleName: "did-you-know",
|
||||||
|
|
@ -683,6 +552,7 @@ func TestExtractKVMeta(t *testing.T) {
|
||||||
Team: "a-team",
|
Team: "a-team",
|
||||||
Version: "old",
|
Version: "old",
|
||||||
Language: "jive",
|
Language: "jive",
|
||||||
|
Names: []string{"last-call", "watch-out-now"},
|
||||||
ExpectedAlertRoutes: []AlertRoute{
|
ExpectedAlertRoutes: []AlertRoute{
|
||||||
{
|
{
|
||||||
RuleName: "last-call",
|
RuleName: "last-call",
|
||||||
|
|
@ -729,6 +599,9 @@ func TestExtractKVMeta(t *testing.T) {
|
||||||
Team: "diversity",
|
Team: "diversity",
|
||||||
Version: "kv-routes",
|
Version: "kv-routes",
|
||||||
Language: "understanding",
|
Language: "understanding",
|
||||||
|
Names: []string{
|
||||||
|
"all-combos", "there's-app-invites-everywhere", "what's-the-catch", "last-call",
|
||||||
|
},
|
||||||
ExpectedMetricsRoutes: []MetricsRoute{
|
ExpectedMetricsRoutes: []MetricsRoute{
|
||||||
{
|
{
|
||||||
RuleName: "all-combos",
|
RuleName: "all-combos",
|
||||||
|
|
@ -807,6 +680,10 @@ func TestExtractKVMeta(t *testing.T) {
|
||||||
assert.Equal(test.Language, kvmeta.Language)
|
assert.Equal(test.Language, kvmeta.Language)
|
||||||
assert.Equal(test.Version, kvmeta.Version)
|
assert.Equal(test.Version, kvmeta.Version)
|
||||||
|
|
||||||
|
assert.EqualValues(
|
||||||
|
sort.StringSlice(test.Names), sort.StringSlice(kvmeta.Routes.RuleNames()),
|
||||||
|
)
|
||||||
|
|
||||||
assert.Equal(len(test.ExpectedMetricsRoutes), len(kvmeta.Routes.MetricsRoutes()))
|
assert.Equal(len(test.ExpectedMetricsRoutes), len(kvmeta.Routes.MetricsRoutes()))
|
||||||
for idx, route := range kvmeta.Routes.MetricsRoutes() {
|
for idx, route := range kvmeta.Routes.MetricsRoutes() {
|
||||||
expected := test.ExpectedMetricsRoutes[idx]
|
expected := test.ExpectedMetricsRoutes[idx]
|
||||||
|
|
@ -853,7 +730,7 @@ func BenchmarkFieldsFromSyslog(b *testing.B) {
|
||||||
|
|
||||||
func BenchmarkParseAndEnhance(b *testing.B) {
|
func BenchmarkParseAndEnhance(b *testing.B) {
|
||||||
for n := 0; n < b.N; n++ {
|
for n := 0; n < b.N; n++ {
|
||||||
_, err := ParseAndEnhance(benchmarkLine, "env", false, false, time.Time{})
|
_, err := ParseAndEnhance(benchmarkLine, "env")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
b.FailNow()
|
b.FailNow()
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue