diff --git a/decode/decode_test.go b/decode/decode_test.go index f71ea1f..12136c0 100644 --- a/decode/decode_test.go +++ b/decode/decode_test.go @@ -184,6 +184,18 @@ func TestSyslogDecoding(t *testing.T) { }, ExpectedError: nil, }, + Spec{ + Title: "Parses Rsyslog_ FileFormat without PID", + Input: `2017-04-05T21:57:46.794862+00:00 ip-10-0-0-0 env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef: 2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished"}`, + ExpectedOutput: map[string]interface{}{ + "timestamp": logTime3, + "hostname": "ip-10-0-0-0", + "programname": `env--app/arn%3Aaws%3Aecs%3Aus-west-1%3A999988887777%3Atask%2Fabcd1234-1a3b-1a3b-1234-d76552f4b7ef`, + "rawlog": `2017/04/05 21:57:46 some_file.go:10: {"title":"request_finished"}`, + "decoder_msg_type": "syslog", + }, + ExpectedError: nil, + }, Spec{ Title: "Fails to parse non-RSyslog log line", Input: `not rsyslog`, diff --git a/splitter/splitter.go b/splitter/splitter.go index 5d259a7..ba94964 100644 --- a/splitter/splitter.go +++ b/splitter/splitter.go @@ -121,16 +121,15 @@ type RSysLogMessage struct { Timestamp time.Time Hostname string ProgramName string - PID int Message string } func (r RSysLogMessage) String() string { // Adding an extra Microsecond forces `Format` to include all 6 digits within the micorsecond format. // Otherwise, time.Format omits trailing zeroes. (https://github.com/golang/go/issues/12472) - return fmt.Sprintf(`%s %s %s[%d]: %s`, + return fmt.Sprintf(`%s %s %s: %s`, r.Timestamp.Add(time.Microsecond).Format(RFC3339Micro), - r.Hostname, r.ProgramName, r.PID, r.Message) + r.Hostname, r.ProgramName, r.Message) } func splitAWSBatch(b LogEventBatch) ([]RSysLogMessage, bool) { @@ -147,7 +146,6 @@ func splitAWSBatch(b LogEventBatch) ([]RSysLogMessage, bool) { out = append(out, RSysLogMessage{ Timestamp: event.Timestamp.Time(), ProgramName: env + "--" + app + arnCruft + task, - PID: 1, Hostname: "aws-batch", Message: event.Message, }) @@ -175,7 +173,6 @@ func splitAWSLambda(b LogEventBatch) ([]RSysLogMessage, bool) { out = append(out, RSysLogMessage{ Timestamp: event.Timestamp.Time(), ProgramName: env + "--" + app + arnCruft + task, - PID: 1, Hostname: "aws-lambda", Message: event.Message, }) @@ -202,7 +199,6 @@ func splitAWSFargate(b LogEventBatch) ([]RSysLogMessage, bool) { out = append(out, RSysLogMessage{ Timestamp: event.Timestamp.Time(), ProgramName: env + "--" + app + arnCruft + ecsTaskID, - PID: 1, Hostname: "aws-fargate", Message: event.Message, }) @@ -217,7 +213,6 @@ func splitDefault(b LogEventBatch) []RSysLogMessage { Timestamp: event.Timestamp.Time(), Hostname: b.LogStream, ProgramName: b.LogGroup + "--" + b.LogStream + arnCruft + taskCruft, - PID: 1, Message: event.Message, }) } diff --git a/splitter/splitter_test.go b/splitter/splitter_test.go index ccc141a..a087325 100644 --- a/splitter/splitter_test.go +++ b/splitter/splitter_test.go @@ -122,8 +122,8 @@ func TestSplitBatch(t *testing.T) { } lines := Split(input) expected := [][]byte{ - []byte("2017-06-26T23:32:23.285001+00:00 aws-batch env--app/arn%3Aaws%3Aecs%3Aus-east-1%3A999988887777%3Atask%2F12345678-1234-1234-1234-555566667777[1]: some log line"), - []byte("2017-06-26T23:32:23.285001+00:00 aws-batch env--app/arn%3Aaws%3Aecs%3Aus-east-1%3A999988887777%3Atask%2F12345678-1234-1234-1234-555566667777[1]: another log line"), + []byte("2017-06-26T23:32:23.285001+00:00 aws-batch env--app/arn%3Aaws%3Aecs%3Aus-east-1%3A999988887777%3Atask%2F12345678-1234-1234-1234-555566667777: some log line"), + []byte("2017-06-26T23:32:23.285001+00:00 aws-batch env--app/arn%3Aaws%3Aecs%3Aus-east-1%3A999988887777%3Atask%2F12345678-1234-1234-1234-555566667777: another log line"), } assert.Equal(t, expected, lines) } @@ -155,9 +155,9 @@ func TestSplitLambda(t *testing.T) { } lines := Split(input) expected := [][]byte{ - []byte(`2017-06-26T23:32:23.285001+00:00 aws-lambda env--app/arn%3Aaws%3Aecs%3Aus-east-1%3A999988887777%3Atask%2F8edbd53f-64c7-4a3c-bf1e-efeff40f6512[1]: START RequestId: 8edbd53f-64c7-4a3c-bf1e-efeff40f6512 Version: 3`), - []byte(`2017-06-26T23:32:23.285001+00:00 aws-lambda env--app/arn%3Aaws%3Aecs%3Aus-east-1%3A999988887777%3Atask%2F8edbd53f-64c7-4a3c-bf1e-efeff40f6512[1]: {"aws_request_id":"8edbd53f-64c7-4a3c-bf1e-efeff40f6512","level":"info","source":"app","title":"some-log-title"}`), - []byte(`2017-06-26T23:32:23.285001+00:00 aws-lambda env--app/arn%3Aaws%3Aecs%3Aus-east-1%3A999988887777%3Atask%2F12345678-1234-1234-1234-555566667777[1]: Example message that doesn't contain a request ID`), + []byte(`2017-06-26T23:32:23.285001+00:00 aws-lambda env--app/arn%3Aaws%3Aecs%3Aus-east-1%3A999988887777%3Atask%2F8edbd53f-64c7-4a3c-bf1e-efeff40f6512: START RequestId: 8edbd53f-64c7-4a3c-bf1e-efeff40f6512 Version: 3`), + []byte(`2017-06-26T23:32:23.285001+00:00 aws-lambda env--app/arn%3Aaws%3Aecs%3Aus-east-1%3A999988887777%3Atask%2F8edbd53f-64c7-4a3c-bf1e-efeff40f6512: {"aws_request_id":"8edbd53f-64c7-4a3c-bf1e-efeff40f6512","level":"info","source":"app","title":"some-log-title"}`), + []byte(`2017-06-26T23:32:23.285001+00:00 aws-lambda env--app/arn%3Aaws%3Aecs%3Aus-east-1%3A999988887777%3Atask%2F12345678-1234-1234-1234-555566667777: Example message that doesn't contain a request ID`), } assert.Equal(t, expected, lines) for i, line := range expected { @@ -190,7 +190,7 @@ func TestSplitFargate(t *testing.T) { } lines := Split(input) expected := [][]byte{ - []byte(`2017-06-26T23:32:23.285001+00:00 aws-fargate production--clever-com-router/arn%3Aaws%3Aecs%3Aus-east-1%3A999988887777%3Atask%2F27b22d5d68aa4bd3923c95e7f32a3852[1]: Starting haproxy: haproxy.`), + []byte(`2017-06-26T23:32:23.285001+00:00 aws-fargate production--clever-com-router/arn%3Aaws%3Aecs%3Aus-east-1%3A999988887777%3Atask%2F27b22d5d68aa4bd3923c95e7f32a3852: Starting haproxy: haproxy.`), } assert.Equal(t, expected, lines) for _, line := range expected { @@ -220,7 +220,7 @@ func TestSplitDefault(t *testing.T) { } lines := Split(input) expected := [][]byte{ - []byte(`2017-06-26T23:32:23.285001+00:00 eni-43403819-all vpn_flow_logs--eni-43403819-all/arn%3Aaws%3Aecs%3Aus-east-1%3A999988887777%3Atask%2F12345678-1234-1234-1234-555566667777[1]: 2 589690932525 eni-43403819 10.0.0.233 172.217.6.46 64067 443 17 8 3969 1516891809 1516891868 ACCEPT OK`), + []byte(`2017-06-26T23:32:23.285001+00:00 eni-43403819-all vpn_flow_logs--eni-43403819-all/arn%3Aaws%3Aecs%3Aus-east-1%3A999988887777%3Atask%2F12345678-1234-1234-1234-555566667777: 2 589690932525 eni-43403819 10.0.0.233 172.217.6.46 64067 443 17 8 3969 1516891809 1516891868 ACCEPT OK`), } assert.Equal(t, expected, lines) }