luciano/amazon-kinesis-client-go
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Added decoder_msg_type field to logs parsed by syslog

Browse source
This commit is contained in:
Xavi Ramirez 2018-05-10 23:56:14 +00:00
parent 3ded2fcd2d
commit 74634a6bf0
2 changed files with 38 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
decode/decode.go
View file

@ -57,6 +57,9 @@ func FieldsFromSyslog(line string) (map[string]interface{}, error) {
out[newKey] = v
}
}
out["decoder_msg_type"] = "syslog"
return out, nil
}

64
decode/decode_test.go
View file

@ -127,10 +127,11 @@ func TestSyslogDecoding(t *testing.T) {
Title: "Parses Rsyslog_TraditionalFileFormat with simple log body",
Input: `Oct 25 10:20:37 some-host docker/fa3a5e338a47[1294]: log body`,
ExpectedOutput: map[string]interface{}{
"timestamp": logTime,
"hostname": "some-host",
"programname": "docker/fa3a5e338a47",
"rawlog": "log body",
"timestamp": logTime,
"hostname": "some-host",
"programname": "docker/fa3a5e338a47",
"rawlog": "log body",
"decoder_msg_type": "syslog",
},
ExpectedError: nil,
},
@ -138,10 +139,11 @@ func TestSyslogDecoding(t *testing.T) {
Title: "Parses Rsyslog_TraditionalFileFormat with haproxy access log body",
Input: `Apr 5 21:45:54 influx-service docker/0000aa112233[1234]: [httpd] 2017/04/05 21:45:54 172.17.42.1 - heka [05/Apr/2017:21:45:54 +0000] POST /write?db=foo&precision=ms HTTP/1.1 204 0 - Go 1.1 package http 123456-1234-1234-b11b-000000000000 13.688672ms`,
ExpectedOutput: map[string]interface{}{
"timestamp": logTime2,
"hostname": "influx-service",
"programname": "docker/0000aa112233",
"rawlog": "[httpd] 2017/04/05 21:45:54 172.17.42.1 - heka [05/Apr/2017:21:45:54 +0000] POST /write?db=foo&precision=ms HTTP/1.1 204 0 - Go 1.1 package http 123456-1234-1234-b11b-000000000000 13.688672ms",
"timestamp": logTime2,
"hostname": "influx-service",
"programname": "docker/0000aa112233",
"rawlog": "[httpd] 2017/04/05 21:45:54 172.17.42.1 - heka [05/Apr/2017:21:45:54 +0000] POST /write?db=foo&precision=ms HTTP/1.1 204 0 - Go 1.1 package http 123456-1234-1234-b11b-000000000000 13.688672ms",
"decoder_msg_type": "syslog",
},
ExpectedError: nil,
},
@ -149,10 +151,11 @@ func TestSyslogDecoding(t *testing.T) {
Title: "Parses Rsyslog_TraditionalFileFormat",
Input: `Apr 5 21:45:54 mongodb-some-machine whackanop: 2017/04/05 21:46:11 found 0 ops`,
ExpectedOutput: map[string]interface{}{
"timestamp": logTime2,
"hostname": "mongodb-some-machine",
"programname": "whackanop",
"rawlog": "2017/04/05 21:46:11 found 0 ops",
"timestamp": logTime2,
"hostname": "mongodb-some-machine",
"programname": "whackanop",
"rawlog": "2017/04/05 21:46:11 found 0 ops",
"decoder_msg_type": "syslog",
},
ExpectedError: nil,
},
@ -160,10 +163,11 @@ func TestSyslogDecoding(t *testing.T) {
Title: "Parses Rsyslog_ FileFormat with Kayvee payload",
Input: `2017-04-05T21:57:46.794862+00:00 ip-10-0-0-0 env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef[3291]: 2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished"}`,
ExpectedOutput: map[string]interface{}{
"timestamp": logTime3,
"hostname": "ip-10-0-0-0",
"programname": `env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef`,
"rawlog": `2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished"}`,
"timestamp": logTime3,
"hostname": "ip-10-0-0-0",
"programname": `env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef`,
"rawlog": `2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished"}`,
"decoder_msg_type": "syslog",
},
ExpectedError: nil,
},
@ -253,14 +257,15 @@ func TestParseAndEnhance(t *testing.T) {
Title: "Parses a non-Kayvee log line",
Line: `2017-04-05T21:57:46.794862+00:00 ip-10-0-0-0 env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef[3291]: some log`,
ExpectedOutput: map[string]interface{}{
"timestamp": logTime3,
"hostname": "ip-10-0-0-0",
"programname": `env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef`,
"rawlog": `some log`,
"env": "deploy-env",
"container_env": "env",
"container_app": "app",
"container_task": "abcd1234-1a3b-1a3b-1234-d76552f4b7ef",
"timestamp": logTime3,
"hostname": "ip-10-0-0-0",
"programname": `env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef`,
"rawlog": `some log`,
"env": "deploy-env",
"decoder_msg_type": "syslog",
"container_env": "env",
"container_app": "app",
"container_task": "abcd1234-1a3b-1a3b-1234-d76552f4b7ef",
},
ExpectedError: nil,
},
@ -294,11 +299,12 @@ func TestParseAndEnhance(t *testing.T) {
Title: "Log with timestamp time.RFC3339 format",
Line: `2017-04-05T21:57:46+00:00 mongo-docker-pipeline-r10-4 diamond[24099] Signal Received: 15`,
ExpectedOutput: map[string]interface{}{
"env": "deploy-env",
"hostname": "mongo-docker-pipeline-r10-4",
"programname": "diamond",
"rawlog": "Signal Received: 15",
"timestamp": logTime2,
"env": "deploy-env",
"hostname": "mongo-docker-pipeline-r10-4",
"programname": "diamond",
"decoder_msg_type": "syslog",
"rawlog": "Signal Received: 15",
"timestamp": logTime2,
},
},
}