diff --git a/internal/generator/vector/output/common/template/template.go b/internal/generator/vector/output/common/template/template.go index b6ed246614..784326fb03 100644 --- a/internal/generator/vector/output/common/template/template.go +++ b/internal/generator/vector/output/common/template/template.go @@ -47,9 +47,12 @@ func TemplateRemap(componentID string, inputs []string, userTemplate, field, des // TransformUserTemplateToVRL converts the user entered template to VRL compatible syntax // Example: foo-{.log_type||"none"} -> "foo-" + to_string!(.log_type||"none") -func TransformUserTemplateToVRL(userTemplate string) string { +func TransformUserTemplateToVRL(userTemplate string, suffix ...string) string { // Finds and replaces expressions defined in `{}` with to_string!() replacedUserTemplate := ReplaceBracketWithToString(userTemplate, "to_string!(%s)") + if len(suffix) > 0 && suffix[0] != "" { + replacedUserTemplate = ReplaceBracketWithToString(userTemplate, "to_string!("+suffix[0]+"%s)") + } // Finding all matches of to_string!() returning their start + end indices matchedIndices := splitRegex.FindAllStringSubmatchIndex(replacedUserTemplate, -1) diff --git a/internal/generator/vector/output/syslog/rfc3164_with_defaults.toml b/internal/generator/vector/output/syslog/rfc3164_with_defaults.toml index a3cd5aa2ab..bd04cd09fb 100644 --- a/internal/generator/vector/output/syslog/rfc3164_with_defaults.toml +++ b/internal/generator/vector/output/syslog/rfc3164_with_defaults.toml @@ -2,25 +2,25 @@ type = "remap" inputs = ["application"] source = ''' -. = merge(., parse_json!(string!(.message))) ?? . if .log_type == "infrastructure" && .log_source == "node" { ._internal.syslog.tag = to_string!(.systemd.u.SYSLOG_IDENTIFIER || "") ._internal.syslog.proc_id = to_string!(.systemd.t.PID || "") } if .log_source == "container" { ._internal.syslog.tag = join!([.kubernetes.namespace_name, .kubernetes.pod_name, .kubernetes.container_name], "") - ._internal.syslog.severity = .level - ._internal.syslog.facility = "user" - #Remove non-alphanumeric characters + #Remove non-alphanumeric characters ._internal.syslog.tag = replace(._internal.syslog.tag, r'[^a-zA-Z0-9]', "") #Truncate the sanitized tag to 32 characters ._internal.syslog.tag = truncate(._internal.syslog.tag, 32) + ._internal.syslog.severity = .level + ._internal.syslog.facility = "user" } if .log_type == "audit" { ._internal.syslog.tag = .log_source ._internal.syslog.severity = "informational" ._internal.syslog.facility = "security" } + .facility = to_string!(._internal.syslog.facility || "user") .severity = to_string!(._internal.syslog.severity || "informational") .proc_id = to_string!(._internal.syslog.proc_id || "-") diff --git a/internal/generator/vector/output/syslog/syslog.go b/internal/generator/vector/output/syslog/syslog.go index 714fa5e423..a3d0c04d2f 100644 --- a/internal/generator/vector/output/syslog/syslog.go +++ b/internal/generator/vector/output/syslog/syslog.go @@ -20,8 +20,42 @@ import ( ) const ( - TCP = `tcp` - TLS = `tls` + TCP = `tcp` + TLS = `tls` + ParsedMsg = "parsed_msg" + defFacility = `{._internal.syslog.facility || "user"}` + defSeverity = `{._internal.syslog.severity || "informational"}` + defProcId = `{._internal.syslog.proc_id || "-"}` + defTag = `{._internal.syslog.tag || ""}` + defAppName = `{._internal.syslog.app_name || "-"}` + defMsgId = `{._internal.syslog.msg_id || "-"}` + + nodeTag = `._internal.syslog.tag = to_string!(.systemd.u.SYSLOG_IDENTIFIER || "")` + nodeProcId = `._internal.syslog.proc_id = to_string!(.systemd.t.PID || "")` + + containerTag = `._internal.syslog.tag = join!([.kubernetes.namespace_name, .kubernetes.pod_name, .kubernetes.container_name], "") +#Remove non-alphanumeric characters +._internal.syslog.tag = replace(._internal.syslog.tag, r'[^a-zA-Z0-9]', "") +#Truncate the sanitized tag to 32 characters +._internal.syslog.tag = truncate(._internal.syslog.tag, 32) +` + containerSeverity = `._internal.syslog.severity = .level` + containerFacility = `._internal.syslog.facility = "user"` + auditTag = `._internal.syslog.tag = .log_source` + auditSeverity = `._internal.syslog.severity = "informational"` + auditFacility = `._internal.syslog.facility = "security"` + msgId = `._internal.syslog.msg_id = .log_source` + nodeAppName = `._internal.syslog.app_name = to_string!(.systemd.u.SYSLOG_IDENTIFIER||"-")` + nodeProcId2 = `._internal.syslog.proc_id = to_string!(.systemd.t.PID||"-")` + containerAppName = `._internal.syslog.app_name = join!([.kubernetes.namespace_name, .kubernetes.pod_name, .kubernetes.container_name], "_")` + containerProcId = `._internal.syslog.proc_id = to_string!(.kubernetes.pod_id||"-")` + containerSeverity2 = `._internal.syslog.severity = .level` + containerFacility2 = `._internal.syslog.facility = "user"` + auditAppName = `._internal.syslog.app_name = .log_source` + auditProcId2 = `._internal.syslog.proc_id = to_string!(.auditID || "-")` + + auditSeverity2 = `._internal.syslog.severity = "informational"` + auditFacility2 = `._internal.syslog.facility = "security"` ) type Syslog struct { @@ -61,6 +95,8 @@ type SyslogEncodingRemap struct { EncodingFields EncodingTemplateField PayloadKey string RFC string + Defaults string + ParseMsg bool } func (ser SyslogEncodingRemap) Name() string { @@ -68,57 +104,25 @@ func (ser SyslogEncodingRemap) Name() string { } func (ser SyslogEncodingRemap) Template() string { - return `{{define "` + ser.Name() + `" -}} + return fmt.Sprintf(`{{define "`+ser.Name()+`" -}} [transforms.{{.ComponentID}}] type = "remap" inputs = {{.Inputs}} source = ''' -. = merge(., parse_json!(string!(.message))) ?? . - -{{if eq .RFC "RFC3164" -}} -if .log_type == "infrastructure" && .log_source == "node" { - ._internal.syslog.tag = to_string!(.systemd.u.SYSLOG_IDENTIFIER || "") - ._internal.syslog.proc_id = to_string!(.systemd.t.PID || "") -} -if .log_source == "container" { - ._internal.syslog.tag = join!([.kubernetes.namespace_name, .kubernetes.pod_name, .kubernetes.container_name], "") - ._internal.syslog.severity = .level - ._internal.syslog.facility = "user" - #Remove non-alphanumeric characters - ._internal.syslog.tag = replace(._internal.syslog.tag, r'[^a-zA-Z0-9]', "") - #Truncate the sanitized tag to 32 characters - ._internal.syslog.tag = truncate(._internal.syslog.tag, 32) - -} -if .log_type == "audit" { - ._internal.syslog.tag = .log_source - ._internal.syslog.severity = "informational" - ._internal.syslog.facility = "security" -} +{{if .Defaults}} +{{.Defaults}} {{end}} - -{{if eq .RFC "RFC5424" -}} -._internal.syslog.msg_id = .log_source - -if .log_type == "infrastructure" && .log_source == "node" { - ._internal.syslog.app_name = to_string!(.systemd.u.SYSLOG_IDENTIFIER||"-") - ._internal.syslog.proc_id = to_string!(.systemd.t.PID||"-") -} -if .log_source == "container" { - ._internal.syslog.app_name = join!([.kubernetes.namespace_name, .kubernetes.pod_name, .kubernetes.container_name], "_") - ._internal.syslog.proc_id = to_string!(.kubernetes.pod_id||"-") - ._internal.syslog.severity = .level - ._internal.syslog.facility = "user" -} -if .log_type == "audit" { - ._internal.syslog.app_name = .log_source - ._internal.syslog.proc_id = to_string!(.auditID || "-") - ._internal.syslog.severity = "informational" - ._internal.syslog.facility = "security" +{{if .EncodingFields.FieldVRLList -}} +{{if .ParseMsg}} +_tmp, err = parse_json(string!(.message)) +if err != null { + _tmp = . + log(err, level: "error") +} else { + _tmp = merge!(.,_tmp) } +%s = _tmp {{end}} - -{{if .EncodingFields.FieldVRLList -}} {{range $templatePair := .EncodingFields.FieldVRLList -}} .{{$templatePair.Field}} = {{$templatePair.VRLString}} {{end -}} @@ -139,7 +143,7 @@ if is_null({{.PayloadKey}}) { {{end}} ''' {{end -}} -` +`, ParsedMsg) } type SyslogEncoding struct { @@ -177,7 +181,7 @@ func New(id string, o obs.OutputSpec, inputs []string, secrets observability.Sec } } parseEncodingID := vectorhelpers.MakeID(id, "parse_encoding") - templateFieldPairs := getEncodingTemplatesAndFields(*o.Syslog) + templateFieldPairs, needToParseMsg := getEncodingTemplatesAndFields(*o.Syslog) u, _ := url.Parse(o.Syslog.URL) sink := Output(id, o, []string{parseEncodingID}, secrets, op, u.Scheme, u.Host) if strategy != nil { @@ -185,7 +189,7 @@ func New(id string, o obs.OutputSpec, inputs []string, secrets observability.Sec } syslogElements := []Element{ - parseEncoding(parseEncodingID, inputs, templateFieldPairs, o.Syslog), + parseEncoding(parseEncodingID, inputs, templateFieldPairs, needToParseMsg, o.Syslog), sink, } @@ -214,63 +218,42 @@ func Output(id string, o obs.OutputSpec, inputs []string, secrets observability. // getEncodingTemplatesAndFields determines which encoding fields are templated // so that the templates can be parsed to appropriate VRL -func getEncodingTemplatesAndFields(s obs.Syslog) EncodingTemplateField { +func getEncodingTemplatesAndFields(s obs.Syslog) (EncodingTemplateField, bool) { templateFields := EncodingTemplateField{ FieldVRLList: []FieldVRLStringPair{}, } - if s.Facility == "" { - s.Facility = `{._internal.syslog.facility || "user"}` - } - templateFields.FieldVRLList = append(templateFields.FieldVRLList, FieldVRLStringPair{ - Field: "facility", - VRLString: commontemplate.TransformUserTemplateToVRL(s.Facility), - }) - - if s.Severity == "" { - s.Severity = `{._internal.syslog.severity || "informational"}` + var needToParseMsg = false + + appendField := func(fieldName string, value string, defaultVal string) { + if value == "" { + templateFields.FieldVRLList = append(templateFields.FieldVRLList, FieldVRLStringPair{ + Field: fieldName, + VRLString: commontemplate.TransformUserTemplateToVRL(defaultVal), + }) + } else { + if commontemplate.PathRegex.MatchString(value) { + needToParseMsg = true + } + templateFields.FieldVRLList = append(templateFields.FieldVRLList, FieldVRLStringPair{ + Field: fieldName, + VRLString: commontemplate.TransformUserTemplateToVRL(value, ParsedMsg), + }) + } } - templateFields.FieldVRLList = append(templateFields.FieldVRLList, FieldVRLStringPair{ - Field: "severity", - VRLString: commontemplate.TransformUserTemplateToVRL(s.Severity), - }) - if s.ProcId == "" { - s.ProcId = `{._internal.syslog.proc_id || "-"}` - } - templateFields.FieldVRLList = append(templateFields.FieldVRLList, FieldVRLStringPair{ - Field: "proc_id", - VRLString: commontemplate.TransformUserTemplateToVRL(s.ProcId), - }) + appendField("facility", s.Facility, defFacility) + appendField("severity", s.Severity, defSeverity) + appendField("proc_id", s.ProcId, defProcId) if s.RFC == obs.SyslogRFC3164 { - if s.AppName == "" { - s.AppName = `{._internal.syslog.tag || ""}` - } - templateFields.FieldVRLList = append(templateFields.FieldVRLList, FieldVRLStringPair{ - Field: "tag", - VRLString: commontemplate.TransformUserTemplateToVRL(s.AppName), - }) - + appendField("tag", s.AppName, defTag) } else { - if s.AppName == "" { - s.AppName = `{._internal.syslog.app_name || "-"}` - } - templateFields.FieldVRLList = append(templateFields.FieldVRLList, FieldVRLStringPair{ - Field: "app_name", - VRLString: commontemplate.TransformUserTemplateToVRL(s.AppName), - }) - - if s.MsgId == "" { - s.MsgId = `{._internal.syslog.msg_id || "-"}` - } - templateFields.FieldVRLList = append(templateFields.FieldVRLList, FieldVRLStringPair{ - Field: "msg_id", - VRLString: commontemplate.TransformUserTemplateToVRL(s.MsgId), - }) + appendField("app_name", s.AppName, defAppName) + appendField("msg_id", s.MsgId, defMsgId) } - return templateFields + return templateFields, needToParseMsg } func Encoding(id string, o obs.OutputSpec, templatePairs []FieldVRLStringPair) []Element { @@ -297,13 +280,15 @@ func Encoding(id string, o obs.OutputSpec, templatePairs []FieldVRLStringPair) [ return encodingFields } -func parseEncoding(id string, inputs []string, templatePairs EncodingTemplateField, o *obs.Syslog) Element { +func parseEncoding(id string, inputs []string, templatePairs EncodingTemplateField, needToParseMsg bool, o *obs.Syslog) Element { return SyslogEncodingRemap{ ComponentID: id, Inputs: vectorhelpers.MakeInputs(inputs...), EncodingFields: templatePairs, PayloadKey: PayloadKey(o.PayloadKey), RFC: string(o.RFC), + Defaults: buildDefaults(o), + ParseMsg: needToParseMsg, } } @@ -345,3 +330,90 @@ var keyre = regexp.MustCompile(`^\$(\.[[:word:]]*)+$`) func IsKeyExpr(str string) bool { return keyre.MatchString(str) } + +func buildDefaults(o *obs.Syslog) string { + var builder strings.Builder + type defaultRule struct { + cond string + appName string + procId string + severity string + facility string + } + var defaultRules []defaultRule + + if o.RFC == obs.SyslogRFC3164 { + if o.ProcId == "" || o.AppName == "" { + defaultRules = append(defaultRules, defaultRule{ + cond: `.log_type == "infrastructure" && .log_source == "node"`, + appName: ifEmpty(o.AppName, nodeTag), + procId: ifEmpty(o.ProcId, nodeProcId), + }) + } + if o.AppName == "" || o.Severity == "" || o.Facility == "" { + defaultRules = append(defaultRules, defaultRule{ + cond: `.log_source == "container"`, + appName: ifEmpty(o.AppName, containerTag), + severity: ifEmpty(o.Severity, containerSeverity), + facility: ifEmpty(o.Facility, containerFacility), + }, defaultRule{ + cond: `.log_type == "audit"`, + appName: ifEmpty(o.AppName, auditTag), + severity: ifEmpty(o.Severity, auditSeverity), + facility: ifEmpty(o.Facility, auditFacility), + }) + } + } + + if o.RFC == obs.SyslogRFC5424 { + builder.WriteString(msgId + "\n") + if o.ProcId == "" || o.AppName == "" { + defaultRules = append(defaultRules, defaultRule{ + cond: `.log_type == "infrastructure" && .log_source == "node"`, + appName: ifEmpty(o.AppName, nodeAppName), + procId: ifEmpty(o.ProcId, nodeProcId2), + }) + } + if o.AppName == "" || o.ProcId == "" || o.Severity == "" || o.Facility == "" { + defaultRules = append(defaultRules, defaultRule{ + cond: `.log_source == "container"`, + appName: ifEmpty(o.AppName, containerAppName), + procId: ifEmpty(o.ProcId, containerProcId), + severity: ifEmpty(o.Severity, containerSeverity), + facility: ifEmpty(o.Facility, containerFacility), + }, defaultRule{ + cond: `.log_type == "audit"`, + appName: ifEmpty(o.AppName, auditAppName), + procId: ifEmpty(o.ProcId, auditProcId2), + severity: ifEmpty(o.Severity, auditSeverity2), + facility: ifEmpty(o.Facility, auditFacility2), + }) + } + } + + for _, b := range defaultRules { + builder.WriteString(fmt.Sprintf("if %s {\n", b.cond)) + if b.appName != "" { + builder.WriteString(b.appName + "\n") + } + if b.procId != "" { + builder.WriteString(b.procId + "\n") + } + if b.severity != "" { + builder.WriteString(b.severity + "\n") + } + if b.facility != "" { + builder.WriteString(b.facility + "\n") + } + builder.WriteString("}\n") + } + + return builder.String() +} + +func ifEmpty(val, defaultVal string) string { + if val == "" { + return defaultVal + } + return "" +} diff --git a/internal/generator/vector/output/syslog/tcp_with_defaults.toml b/internal/generator/vector/output/syslog/tcp_with_defaults.toml index 3b28054c70..4dbf9b00f3 100644 --- a/internal/generator/vector/output/syslog/tcp_with_defaults.toml +++ b/internal/generator/vector/output/syslog/tcp_with_defaults.toml @@ -2,7 +2,6 @@ type = "remap" inputs = ["application"] source = ''' -. = merge(., parse_json!(string!(.message))) ?? . ._internal.syslog.msg_id = .log_source if .log_type == "infrastructure" && .log_source == "node" { @@ -21,6 +20,7 @@ if .log_type == "audit" { ._internal.syslog.severity = "informational" ._internal.syslog.facility = "security" } + .facility = to_string!(._internal.syslog.facility || "user") .severity = to_string!(._internal.syslog.severity || "informational") .proc_id = to_string!(._internal.syslog.proc_id || "-") diff --git a/internal/generator/vector/output/syslog/tcp_with_tuning.toml b/internal/generator/vector/output/syslog/tcp_with_tuning.toml index bc92181439..5d464fe9dd 100644 --- a/internal/generator/vector/output/syslog/tcp_with_tuning.toml +++ b/internal/generator/vector/output/syslog/tcp_with_tuning.toml @@ -2,10 +2,7 @@ type = "remap" inputs = ["application"] source = ''' -. = merge(., parse_json!(string!(.message))) ?? . - ._internal.syslog.msg_id = .log_source - if .log_type == "infrastructure" && .log_source == "node" { ._internal.syslog.app_name = to_string!(.systemd.u.SYSLOG_IDENTIFIER||"-") ._internal.syslog.proc_id = to_string!(.systemd.t.PID||"-") @@ -22,6 +19,7 @@ if .log_type == "audit" { ._internal.syslog.severity = "informational" ._internal.syslog.facility = "security" } + .facility = to_string!(._internal.syslog.facility || "user") .severity = to_string!(._internal.syslog.severity || "informational") .proc_id = to_string!(._internal.syslog.proc_id || "-") diff --git a/internal/generator/vector/output/syslog/tls_with_field_references.toml b/internal/generator/vector/output/syslog/tls_with_field_references.toml index d01283a5de..cea87b5b83 100644 --- a/internal/generator/vector/output/syslog/tls_with_field_references.toml +++ b/internal/generator/vector/output/syslog/tls_with_field_references.toml @@ -2,31 +2,21 @@ type = "remap" inputs = ["application"] source = ''' -. = merge(., parse_json!(string!(.message))) ?? . - ._internal.syslog.msg_id = .log_source - -if .log_type == "infrastructure" && .log_source == "node" { - ._internal.syslog.app_name = to_string!(.systemd.u.SYSLOG_IDENTIFIER||"-") - ._internal.syslog.proc_id = to_string!(.systemd.t.PID||"-") -} -if .log_source == "container" { - ._internal.syslog.app_name = join!([.kubernetes.namespace_name, .kubernetes.pod_name, .kubernetes.container_name], "_") - ._internal.syslog.proc_id = to_string!(.kubernetes.pod_id||"-") - ._internal.syslog.severity = .level - ._internal.syslog.facility = "user" -} -if .log_type == "audit" { - ._internal.syslog.app_name = .log_source - ._internal.syslog.proc_id = to_string!(.auditID || "-") - ._internal.syslog.severity = "informational" - ._internal.syslog.facility = "security" +_tmp, err = parse_json(string!(.message)) +if err != null { + _tmp = . + log(err, level: "error") +} else { + _tmp = merge!(.,_tmp) } -.facility = to_string!(.facility||"none") -.severity = to_string!(.severity||"none") -.proc_id = to_string!(.proc_id||"none") -.app_name = to_string!(.app_name||"none") -.msg_id = to_string!(.msg_id||"none") +parsed_msg = _tmp + +.facility = to_string!(parsed_msg.facility||"none") +.severity = to_string!(parsed_msg.severity||"none") +.proc_id = to_string!(parsed_msg.proc_id||"none") +.app_name = to_string!(parsed_msg.app_name||"none") +.msg_id = to_string!(parsed_msg.msg_id||"none") if is_null(.payload_key) { .payload_key = . diff --git a/internal/generator/vector/output/syslog/udp_with_every_setting.toml b/internal/generator/vector/output/syslog/udp_with_every_setting.toml index f531716c9e..1ae2ed1f75 100644 --- a/internal/generator/vector/output/syslog/udp_with_every_setting.toml +++ b/internal/generator/vector/output/syslog/udp_with_every_setting.toml @@ -2,25 +2,6 @@ type = "remap" inputs = ["application"] source = ''' -. = merge(., parse_json!(string!(.message))) ?? . -if .log_type == "infrastructure" && .log_source == "node" { - ._internal.syslog.tag = to_string!(.systemd.u.SYSLOG_IDENTIFIER || "") - ._internal.syslog.proc_id = to_string!(.systemd.t.PID || "") -} -if .log_source == "container" { - ._internal.syslog.tag = join!([.kubernetes.namespace_name, .kubernetes.pod_name, .kubernetes.container_name], "") - ._internal.syslog.severity = .level - ._internal.syslog.facility = "user" - #Remove non-alphanumeric characters - ._internal.syslog.tag = replace(._internal.syslog.tag, r'[^a-zA-Z0-9]', "") - #Truncate the sanitized tag to 32 characters - ._internal.syslog.tag = truncate(._internal.syslog.tag, 32) -} -if .log_type == "audit" { - ._internal.syslog.tag = .log_source - ._internal.syslog.severity = "informational" - ._internal.syslog.facility = "security" -} .facility = "kern" .severity = "critical" .proc_id = "procID" diff --git a/internal/generator/vector/output/syslog/xyz_defaults.toml b/internal/generator/vector/output/syslog/xyz_defaults.toml index 9d175e4e38..488e781b04 100644 --- a/internal/generator/vector/output/syslog/xyz_defaults.toml +++ b/internal/generator/vector/output/syslog/xyz_defaults.toml @@ -2,7 +2,6 @@ type = "remap" inputs = ["application"] source = ''' -. = merge(., parse_json!(string!(.message))) ?? . ._internal.syslog.msg_id = .log_source if .log_type == "infrastructure" && .log_source == "node" { ._internal.syslog.app_name = to_string!(.systemd.u.SYSLOG_IDENTIFIER||"-")