From d1473f894acb0e47e458218c45af4420b846852c Mon Sep 17 00:00:00 2001 From: Nathan Leiby Date: Thu, 17 Aug 2017 14:36:53 -0700 Subject: [PATCH] decode: s/type/decoder_msg_type type is a really generally name. sometimes, it is used in other libraries or various output as a valid field name. let's not make it a reserved keyword that gets overriden by type == Kayvee --- decode/decode.go | 4 +- decode/decode_test.go | 236 +++++++++++++++++++++--------------------- 2 files changed, 120 insertions(+), 120 deletions(-) diff --git a/decode/decode.go b/decode/decode.go index dd8666d..103a405 100644 --- a/decode/decode.go +++ b/decode/decode.go @@ -15,7 +15,7 @@ import ( var reservedFields = []string{ "prefix", "postfix", - "Type", + "decoder_msg_type", } func stringInSlice(s string, slice []string) bool { @@ -97,7 +97,7 @@ func FieldsFromKayvee(line string) (map[string]interface{}, error) { } } - m["type"] = "Kayvee" + m["decoder_msg_type"] = "Kayvee" return m, nil } diff --git a/decode/decode_test.go b/decode/decode_test.go index fd05f21..9e64736 100644 --- a/decode/decode_test.go +++ b/decode/decode_test.go @@ -24,10 +24,10 @@ func TestKayveeDecoding(t *testing.T) { Title: "handles just JSON", Input: `{"a":"b"}`, ExpectedOutput: map[string]interface{}{ - "prefix": "", - "postfix": "", - "a": "b", - "type": "Kayvee", + "prefix": "", + "postfix": "", + "a": "b", + "decoder_msg_type": "Kayvee", }, ExpectedError: nil, }, @@ -35,10 +35,10 @@ func TestKayveeDecoding(t *testing.T) { Title: "handles prefix + JSON", Input: `prefix {"a":"b"}`, ExpectedOutput: map[string]interface{}{ - "prefix": "prefix ", - "postfix": "", - "a": "b", - "type": "Kayvee", + "prefix": "prefix ", + "postfix": "", + "a": "b", + "decoder_msg_type": "Kayvee", }, ExpectedError: nil, }, @@ -46,10 +46,10 @@ func TestKayveeDecoding(t *testing.T) { Title: "handles JSON + postfix", Input: `{"a":"b"} postfix`, ExpectedOutput: map[string]interface{}{ - "prefix": "", - "postfix": " postfix", - "a": "b", - "type": "Kayvee", + "prefix": "", + "postfix": " postfix", + "a": "b", + "decoder_msg_type": "Kayvee", }, ExpectedError: nil, }, @@ -57,10 +57,10 @@ func TestKayveeDecoding(t *testing.T) { Title: "handles prefix + JSON + postfix", Input: `prefix {"a":"b"} postfix`, ExpectedOutput: map[string]interface{}{ - "prefix": "prefix ", - "postfix": " postfix", - "a": "b", - "type": "Kayvee", + "prefix": "prefix ", + "postfix": " postfix", + "a": "b", + "decoder_msg_type": "Kayvee", }, ExpectedError: nil, }, @@ -221,18 +221,18 @@ func TestParseAndEnhance(t *testing.T) { Title: "Parses a Kayvee log line from an ECS app", Input: ParseAndEnhanceInput{Line: `2017-04-05T21:57:46.794862+00:00 ip-10-0-0-0 env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef[3291]: 2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished"}`}, ExpectedOutput: map[string]interface{}{ - "timestamp": logTime3, - "hostname": "ip-10-0-0-0", - "programname": `env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef`, - "rawlog": `2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished"}`, - "title": "request_finished", - "type": "Kayvee", - "prefix": "2017/04/05 21:57:46 some_file.go:10: ", - "postfix": "", - "env": "deploy-env", - "container_env": "env", - "container_app": "app", - "container_task": "abcd1234-1a3b-1a3b-1234-d76552f4b7ef", + "timestamp": logTime3, + "hostname": "ip-10-0-0-0", + "programname": `env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef`, + "rawlog": `2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished"}`, + "title": "request_finished", + "decoder_msg_type": "Kayvee", + "prefix": "2017/04/05 21:57:46 some_file.go:10: ", + "postfix": "", + "env": "deploy-env", + "container_env": "env", + "container_app": "app", + "container_task": "abcd1234-1a3b-1a3b-1234-d76552f4b7ef", }, ExpectedError: nil, }, @@ -240,18 +240,18 @@ func TestParseAndEnhance(t *testing.T) { Title: "Parses a Kayvee log line from an ECS app, with override to container_app", Input: ParseAndEnhanceInput{Line: `2017-04-05T21:57:46.794862+00:00 ip-10-0-0-0 env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef[3291]: 2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished","container_app":"force-app"}`}, ExpectedOutput: map[string]interface{}{ - "timestamp": logTime3, - "hostname": "ip-10-0-0-0", - "programname": `env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef`, - "rawlog": `2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished","container_app":"force-app"}`, - "title": "request_finished", - "type": "Kayvee", - "prefix": "2017/04/05 21:57:46 some_file.go:10: ", - "postfix": "", - "env": "deploy-env", - "container_env": "env", - "container_app": "force-app", - "container_task": "abcd1234-1a3b-1a3b-1234-d76552f4b7ef", + "timestamp": logTime3, + "hostname": "ip-10-0-0-0", + "programname": `env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef`, + "rawlog": `2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished","container_app":"force-app"}`, + "title": "request_finished", + "decoder_msg_type": "Kayvee", + "prefix": "2017/04/05 21:57:46 some_file.go:10: ", + "postfix": "", + "env": "deploy-env", + "container_env": "env", + "container_app": "force-app", + "container_task": "abcd1234-1a3b-1a3b-1234-d76552f4b7ef", }, ExpectedError: nil, }, @@ -280,19 +280,19 @@ func TestParseAndEnhance(t *testing.T) { Title: "Parses JSON values", Input: ParseAndEnhanceInput{Line: `2017-04-05T21:57:46.794862+00:00 ip-10-0-0-0 env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef[3291]: 2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished", "nested": {"a":"b"}}`}, ExpectedOutput: map[string]interface{}{ - "timestamp": logTime3, - "hostname": "ip-10-0-0-0", - "programname": `env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef`, - "rawlog": `2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished", "nested": {"a":"b"}}`, - "title": "request_finished", - "type": "Kayvee", - "prefix": "2017/04/05 21:57:46 some_file.go:10: ", - "postfix": "", - "env": "deploy-env", - "container_env": "env", - "container_app": "app", - "container_task": "abcd1234-1a3b-1a3b-1234-d76552f4b7ef", - "nested": map[string]interface{}{"a": "b"}, + "timestamp": logTime3, + "hostname": "ip-10-0-0-0", + "programname": `env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef`, + "rawlog": `2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished", "nested": {"a":"b"}}`, + "title": "request_finished", + "decoder_msg_type": "Kayvee", + "prefix": "2017/04/05 21:57:46 some_file.go:10: ", + "postfix": "", + "env": "deploy-env", + "container_env": "env", + "container_app": "app", + "container_task": "abcd1234-1a3b-1a3b-1234-d76552f4b7ef", + "nested": map[string]interface{}{"a": "b"}, }, ExpectedError: nil, }, @@ -303,19 +303,19 @@ func TestParseAndEnhance(t *testing.T) { StringifyNested: true, }, ExpectedOutput: map[string]interface{}{ - "timestamp": logTime3, - "hostname": "ip-10-0-0-0", - "programname": `env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef`, - "rawlog": `2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished", "nested": {"a":"b"}}`, - "title": "request_finished", - "type": "Kayvee", - "prefix": "2017/04/05 21:57:46 some_file.go:10: ", - "postfix": "", - "env": "deploy-env", - "container_env": "env", - "container_app": "app", - "container_task": "abcd1234-1a3b-1a3b-1234-d76552f4b7ef", - "nested": `{"a":"b"}`, + "timestamp": logTime3, + "hostname": "ip-10-0-0-0", + "programname": `env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef`, + "rawlog": `2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished", "nested": {"a":"b"}}`, + "title": "request_finished", + "decoder_msg_type": "Kayvee", + "prefix": "2017/04/05 21:57:46 some_file.go:10: ", + "postfix": "", + "env": "deploy-env", + "container_env": "env", + "container_app": "app", + "container_task": "abcd1234-1a3b-1a3b-1234-d76552f4b7ef", + "nested": `{"a":"b"}`, }, ExpectedError: nil, }, @@ -326,19 +326,19 @@ func TestParseAndEnhance(t *testing.T) { StringifyNested: true, }, ExpectedOutput: map[string]interface{}{ - "timestamp": logTime3, - "hostname": "ip-10-0-0-0", - "programname": `env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef`, - "rawlog": `2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished", "nested": [{"a":"b"}]}`, - "title": "request_finished", - "type": "Kayvee", - "prefix": "2017/04/05 21:57:46 some_file.go:10: ", - "postfix": "", - "env": "deploy-env", - "container_env": "env", - "container_app": "app", - "container_task": "abcd1234-1a3b-1a3b-1234-d76552f4b7ef", - "nested": `[{"a":"b"}]`, + "timestamp": logTime3, + "hostname": "ip-10-0-0-0", + "programname": `env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef`, + "rawlog": `2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished", "nested": [{"a":"b"}]}`, + "title": "request_finished", + "decoder_msg_type": "Kayvee", + "prefix": "2017/04/05 21:57:46 some_file.go:10: ", + "postfix": "", + "env": "deploy-env", + "container_env": "env", + "container_app": "app", + "container_task": "abcd1234-1a3b-1a3b-1234-d76552f4b7ef", + "nested": `[{"a":"b"}]`, }, ExpectedError: nil, }, @@ -349,19 +349,19 @@ func TestParseAndEnhance(t *testing.T) { RenameESReservedFields: true, }, ExpectedOutput: map[string]interface{}{ - "timestamp": logTime3, - "hostname": "ip-10-0-0-0", - "programname": `env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef`, - "rawlog": `2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished", "_source": "a"}`, - "title": "request_finished", - "type": "Kayvee", - "prefix": "2017/04/05 21:57:46 some_file.go:10: ", - "postfix": "", - "env": "deploy-env", - "container_env": "env", - "container_app": "app", - "container_task": "abcd1234-1a3b-1a3b-1234-d76552f4b7ef", - "kv__source": "a", + "timestamp": logTime3, + "hostname": "ip-10-0-0-0", + "programname": `env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef`, + "rawlog": `2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished", "_source": "a"}`, + "title": "request_finished", + "decoder_msg_type": "Kayvee", + "prefix": "2017/04/05 21:57:46 some_file.go:10: ", + "postfix": "", + "env": "deploy-env", + "container_env": "env", + "container_app": "app", + "container_task": "abcd1234-1a3b-1a3b-1234-d76552f4b7ef", + "kv__source": "a", }, ExpectedError: nil, }, @@ -383,19 +383,19 @@ func TestParseAndEnhance(t *testing.T) { MinimumTimestamp: time.Now().Add(-100 * time.Hour * 24 * 365), // good thru year 2117 }, ExpectedOutput: map[string]interface{}{ - "timestamp": logTime3, - "hostname": "ip-10-0-0-0", - "programname": `env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef`, - "rawlog": `2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished", "_source": "a"}`, - "title": "request_finished", - "type": "Kayvee", - "prefix": "2017/04/05 21:57:46 some_file.go:10: ", - "postfix": "", - "env": "deploy-env", - "container_env": "env", - "container_app": "app", - "container_task": "abcd1234-1a3b-1a3b-1234-d76552f4b7ef", - "kv__source": "a", + "timestamp": logTime3, + "hostname": "ip-10-0-0-0", + "programname": `env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef`, + "rawlog": `2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished", "_source": "a"}`, + "title": "request_finished", + "decoder_msg_type": "Kayvee", + "prefix": "2017/04/05 21:57:46 some_file.go:10: ", + "postfix": "", + "env": "deploy-env", + "container_env": "env", + "container_app": "app", + "container_task": "abcd1234-1a3b-1a3b-1234-d76552f4b7ef", + "kv__source": "a", }, ExpectedError: nil, }, @@ -407,19 +407,19 @@ func TestParseAndEnhance(t *testing.T) { MinimumTimestamp: time.Now().Add(-100 * time.Hour * 24 * 365), // good thru year 2117 }, ExpectedOutput: map[string]interface{}{ - "timestamp": logTime3, - "hostname": "ip-10-0-0-0", - "programname": `env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef`, - "rawlog": `2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished", "_source": "a"}`, - "title": "request_finished", - "type": "Kayvee", - "prefix": "2017/04/05 21:57:46 some_file.go:10: ", - "postfix": "", - "env": "deploy-env", - "container_env": "env", - "container_app": "app", - "container_task": "abcd1234-1a3b-1a3b-1234-d76552f4b7ef", - "kv__source": "a", + "timestamp": logTime3, + "hostname": "ip-10-0-0-0", + "programname": `env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef`, + "rawlog": `2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished", "_source": "a"}`, + "title": "request_finished", + "decoder_msg_type": "Kayvee", + "prefix": "2017/04/05 21:57:46 some_file.go:10: ", + "postfix": "", + "env": "deploy-env", + "container_env": "env", + "container_app": "app", + "container_task": "abcd1234-1a3b-1a3b-1234-d76552f4b7ef", + "kv__source": "a", }, ExpectedError: nil, },