luciano/amazon-kinesis-client-go
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #27 from Clever/more-reserve-fields

Browse source 
Added timestamp, hostname, and rawlog to list of reserved fields
This commit is contained in:
Xavi 2018-05-10 17:09:07 -07:00 committed by GitHub
commit 6e45fa518b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 54 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
decode/decode.go
View file

@ -15,6 +15,9 @@ var reservedFields = []string{
"prefix", "prefix",
"postfix", "postfix",
"decoder_msg_type", "decoder_msg_type",
"timestamp",
"hostname",
"rawlog",
} }
func stringInSlice(s string, slice []string) bool { func stringInSlice(s string, slice []string) bool {
@ -57,6 +60,9 @@ func FieldsFromSyslog(line string) (map[string]interface{}, error) {
out[newKey] = v out[newKey] = v
} }
} }
out["decoder_msg_type"] = "syslog"
return out, nil return out, nil
} }

19
decode/decode_test.go
View file

@ -65,6 +65,19 @@ func TestKayveeDecoding(t *testing.T) {
}, },
ExpectedError: nil, ExpectedError: nil,
}, },
Spec{
Title: "Reserved fields are respected",
Input: `prefix {"a":"b","prefix":"no-override","postfix":"no-override",` +
`"decoder_msg_type":"no-override","timestamp":"no-override",` +
`"hostname":"no-override","rawlog":"no-override"} postfix`,
ExpectedOutput: map[string]interface{}{
"prefix": "prefix ",
"postfix": " postfix",
"a": "b",
"decoder_msg_type": "Kayvee",
},
ExpectedError: nil,
},
Spec{ Spec{
Title: "Returns NonKayveeError if not JSON in body", Title: "Returns NonKayveeError if not JSON in body",
Input: `prefix { postfix`, Input: `prefix { postfix`,
@ -131,6 +144,7 @@ func TestSyslogDecoding(t *testing.T) {
"hostname": "some-host", "hostname": "some-host",
"programname": "docker/fa3a5e338a47", "programname": "docker/fa3a5e338a47",
"rawlog": "log body", "rawlog": "log body",
"decoder_msg_type": "syslog",
}, },
ExpectedError: nil, ExpectedError: nil,
}, },
@ -142,6 +156,7 @@ func TestSyslogDecoding(t *testing.T) {
"hostname": "influx-service", "hostname": "influx-service",
"programname": "docker/0000aa112233", "programname": "docker/0000aa112233",
"rawlog": "[httpd] 2017/04/05 21:45:54 172.17.42.1 - heka [05/Apr/2017:21:45:54 +0000] POST /write?db=foo&precision=ms HTTP/1.1 204 0 - Go 1.1 package http 123456-1234-1234-b11b-000000000000 13.688672ms", "rawlog": "[httpd] 2017/04/05 21:45:54 172.17.42.1 - heka [05/Apr/2017:21:45:54 +0000] POST /write?db=foo&precision=ms HTTP/1.1 204 0 - Go 1.1 package http 123456-1234-1234-b11b-000000000000 13.688672ms",
"decoder_msg_type": "syslog",
}, },
ExpectedError: nil, ExpectedError: nil,
}, },
@ -153,6 +168,7 @@ func TestSyslogDecoding(t *testing.T) {
"hostname": "mongodb-some-machine", "hostname": "mongodb-some-machine",
"programname": "whackanop", "programname": "whackanop",
"rawlog": "2017/04/05 21:46:11 found 0 ops", "rawlog": "2017/04/05 21:46:11 found 0 ops",
"decoder_msg_type": "syslog",
}, },
ExpectedError: nil, ExpectedError: nil,
}, },
@ -164,6 +180,7 @@ func TestSyslogDecoding(t *testing.T) {
"hostname": "ip-10-0-0-0", "hostname": "ip-10-0-0-0",
"programname": `env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef`, "programname": `env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef`,
"rawlog": `2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished"}`, "rawlog": `2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished"}`,
"decoder_msg_type": "syslog",
}, },
ExpectedError: nil, ExpectedError: nil,
}, },
@ -258,6 +275,7 @@ func TestParseAndEnhance(t *testing.T) {
"programname": `env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef`, "programname": `env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef`,
"rawlog": `some log`, "rawlog": `some log`,
"env": "deploy-env", "env": "deploy-env",
"decoder_msg_type": "syslog",
"container_env": "env", "container_env": "env",
"container_app": "app", "container_app": "app",
"container_task": "abcd1234-1a3b-1a3b-1234-d76552f4b7ef", "container_task": "abcd1234-1a3b-1a3b-1234-d76552f4b7ef",
@ -297,6 +315,7 @@ func TestParseAndEnhance(t *testing.T) {
"env": "deploy-env", "env": "deploy-env",
"hostname": "mongo-docker-pipeline-r10-4", "hostname": "mongo-docker-pipeline-r10-4",
"programname": "diamond", "programname": "diamond",
"decoder_msg_type": "syslog",
"rawlog": "Signal Received: 15", "rawlog": "Signal Received: 15",
"timestamp": logTime2, "timestamp": logTime2,
}, },