From 74634a6bf01dfb0a63c3b66e60d9b2f039ba5bd2 Mon Sep 17 00:00:00 2001 From: Xavi Ramirez Date: Thu, 10 May 2018 23:56:14 +0000 Subject: [PATCH 1/2] Added decoder_msg_type field to logs parsed by syslog --- decode/decode.go | 3 ++ decode/decode_test.go | 64 +++++++++++++++++++++++-------------------- 2 files changed, 38 insertions(+), 29 deletions(-) diff --git a/decode/decode.go b/decode/decode.go index 02f3a3b..5fa6914 100644 --- a/decode/decode.go +++ b/decode/decode.go @@ -57,6 +57,9 @@ func FieldsFromSyslog(line string) (map[string]interface{}, error) { out[newKey] = v } } + + out["decoder_msg_type"] = "syslog" + return out, nil } diff --git a/decode/decode_test.go b/decode/decode_test.go index 7e79bce..ba00fc9 100644 --- a/decode/decode_test.go +++ b/decode/decode_test.go @@ -127,10 +127,11 @@ func TestSyslogDecoding(t *testing.T) { Title: "Parses Rsyslog_TraditionalFileFormat with simple log body", Input: `Oct 25 10:20:37 some-host docker/fa3a5e338a47[1294]: log body`, ExpectedOutput: map[string]interface{}{ - "timestamp": logTime, - "hostname": "some-host", - "programname": "docker/fa3a5e338a47", - "rawlog": "log body", + "timestamp": logTime, + "hostname": "some-host", + "programname": "docker/fa3a5e338a47", + "rawlog": "log body", + "decoder_msg_type": "syslog", }, ExpectedError: nil, }, @@ -138,10 +139,11 @@ func TestSyslogDecoding(t *testing.T) { Title: "Parses Rsyslog_TraditionalFileFormat with haproxy access log body", Input: `Apr 5 21:45:54 influx-service docker/0000aa112233[1234]: [httpd] 2017/04/05 21:45:54 172.17.42.1 - heka [05/Apr/2017:21:45:54 +0000] POST /write?db=foo&precision=ms HTTP/1.1 204 0 - Go 1.1 package http 123456-1234-1234-b11b-000000000000 13.688672ms`, ExpectedOutput: map[string]interface{}{ - "timestamp": logTime2, - "hostname": "influx-service", - "programname": "docker/0000aa112233", - "rawlog": "[httpd] 2017/04/05 21:45:54 172.17.42.1 - heka [05/Apr/2017:21:45:54 +0000] POST /write?db=foo&precision=ms HTTP/1.1 204 0 - Go 1.1 package http 123456-1234-1234-b11b-000000000000 13.688672ms", + "timestamp": logTime2, + "hostname": "influx-service", + "programname": "docker/0000aa112233", + "rawlog": "[httpd] 2017/04/05 21:45:54 172.17.42.1 - heka [05/Apr/2017:21:45:54 +0000] POST /write?db=foo&precision=ms HTTP/1.1 204 0 - Go 1.1 package http 123456-1234-1234-b11b-000000000000 13.688672ms", + "decoder_msg_type": "syslog", }, ExpectedError: nil, }, @@ -149,10 +151,11 @@ func TestSyslogDecoding(t *testing.T) { Title: "Parses Rsyslog_TraditionalFileFormat", Input: `Apr 5 21:45:54 mongodb-some-machine whackanop: 2017/04/05 21:46:11 found 0 ops`, ExpectedOutput: map[string]interface{}{ - "timestamp": logTime2, - "hostname": "mongodb-some-machine", - "programname": "whackanop", - "rawlog": "2017/04/05 21:46:11 found 0 ops", + "timestamp": logTime2, + "hostname": "mongodb-some-machine", + "programname": "whackanop", + "rawlog": "2017/04/05 21:46:11 found 0 ops", + "decoder_msg_type": "syslog", }, ExpectedError: nil, }, @@ -160,10 +163,11 @@ func TestSyslogDecoding(t *testing.T) { Title: "Parses Rsyslog_ FileFormat with Kayvee payload", Input: `2017-04-05T21:57:46.794862+00:00 ip-10-0-0-0 env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef[3291]: 2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished"}`, ExpectedOutput: map[string]interface{}{ - "timestamp": logTime3, - "hostname": "ip-10-0-0-0", - "programname": `env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef`, - "rawlog": `2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished"}`, + "timestamp": logTime3, + "hostname": "ip-10-0-0-0", + "programname": `env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef`, + "rawlog": `2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished"}`, + "decoder_msg_type": "syslog", }, ExpectedError: nil, }, @@ -253,14 +257,15 @@ func TestParseAndEnhance(t *testing.T) { Title: "Parses a non-Kayvee log line", Line: `2017-04-05T21:57:46.794862+00:00 ip-10-0-0-0 env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef[3291]: some log`, ExpectedOutput: map[string]interface{}{ - "timestamp": logTime3, - "hostname": "ip-10-0-0-0", - "programname": `env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef`, - "rawlog": `some log`, - "env": "deploy-env", - "container_env": "env", - "container_app": "app", - "container_task": "abcd1234-1a3b-1a3b-1234-d76552f4b7ef", + "timestamp": logTime3, + "hostname": "ip-10-0-0-0", + "programname": `env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef`, + "rawlog": `some log`, + "env": "deploy-env", + "decoder_msg_type": "syslog", + "container_env": "env", + "container_app": "app", + "container_task": "abcd1234-1a3b-1a3b-1234-d76552f4b7ef", }, ExpectedError: nil, }, @@ -294,11 +299,12 @@ func TestParseAndEnhance(t *testing.T) { Title: "Log with timestamp time.RFC3339 format", Line: `2017-04-05T21:57:46+00:00 mongo-docker-pipeline-r10-4 diamond[24099] Signal Received: 15`, ExpectedOutput: map[string]interface{}{ - "env": "deploy-env", - "hostname": "mongo-docker-pipeline-r10-4", - "programname": "diamond", - "rawlog": "Signal Received: 15", - "timestamp": logTime2, + "env": "deploy-env", + "hostname": "mongo-docker-pipeline-r10-4", + "programname": "diamond", + "decoder_msg_type": "syslog", + "rawlog": "Signal Received: 15", + "timestamp": logTime2, }, }, } From 8f2ad09efad66593e1bb39045335e990637259f9 Mon Sep 17 00:00:00 2001 From: Xavi Ramirez Date: Thu, 10 May 2018 23:56:32 +0000 Subject: [PATCH 2/2] Added timestamp, hostname, and rawlog to list of reserved fields --- decode/decode.go | 3 +++ decode/decode_test.go | 13 +++++++++++++ 2 files changed, 16 insertions(+) diff --git a/decode/decode.go b/decode/decode.go index 5fa6914..b0a1874 100644 --- a/decode/decode.go +++ b/decode/decode.go @@ -15,6 +15,9 @@ var reservedFields = []string{ "prefix", "postfix", "decoder_msg_type", + "timestamp", + "hostname", + "rawlog", } func stringInSlice(s string, slice []string) bool { diff --git a/decode/decode_test.go b/decode/decode_test.go index ba00fc9..0b3a28f 100644 --- a/decode/decode_test.go +++ b/decode/decode_test.go @@ -65,6 +65,19 @@ func TestKayveeDecoding(t *testing.T) { }, ExpectedError: nil, }, + Spec{ + Title: "Reserved fields are respected", + Input: `prefix {"a":"b","prefix":"no-override","postfix":"no-override",` + + `"decoder_msg_type":"no-override","timestamp":"no-override",` + + `"hostname":"no-override","rawlog":"no-override"} postfix`, + ExpectedOutput: map[string]interface{}{ + "prefix": "prefix ", + "postfix": " postfix", + "a": "b", + "decoder_msg_type": "Kayvee", + }, + ExpectedError: nil, + }, Spec{ Title: "Returns NonKayveeError if not JSON in body", Input: `prefix { postfix`,