feat(loki.source.syslog): support cisco-specific syslog fields

docs/sources/reference/components/loki/loki.source.syslog.md

@@ -20,9 +20,9 @@ For a detailed example, refer to the [Monitor RFC5424-compliant syslog messages
 If your messages aren't RFC5424 compliant, you can use `raw` syslog format in combination with the [`loki.process`](./loki.process.md) component.
 
 Please note, that the `raw` syslog format is an [experimental][] feature.
-{{< /admonition >}}
 
 [experimental]: https://grafana.com/docs/release-life-cycle/
+{{< /admonition >}}
 
 The component starts a new syslog listener for each of the given `config` blocks and fans out incoming entries to the list of receivers in `forward_to`.
 
@@ -65,6 +65,8 @@ The `relabel_rules` field can make use of the `rules` export value from a [`loki
 * `__syslog_message_app_name`
 * `__syslog_message_proc_id`
 * `__syslog_message_msg_id`
+* `__syslog_message_msg_counter`
+* `__syslog_message_sequence`
 
 If there is [RFC5424](https://www.rfc-editor.org/rfc/rfc5424) compliant structured data in the parsed message, it will be applied to the log entry as a label with prefix `__syslog_message_sd_`.
 For example, if the structured data provided is `[example@99999 test="value"]`, the log entry will have the label `__syslog_message_sd_example_99999_test` with a value of `value`.
@@ -88,18 +90,20 @@ loki.relabel "syslog" {
 
 You can use the following blocks with `loki.source.syslog`:
 
-| Name                                                    | Description                                                                 | Required |
-|---------------------------------------------------------|-----------------------------------------------------------------------------|----------|
-| [`listener`][listener]                                  | Configures a listener for Syslog messages.                                  | no       |
-| `listener` > [`raw_format_options`][raw_format_options] | Configures `raw` syslog format behavior.                                    | no       |
-| `listener` > [`tls_config`][tls_config]                 | Configures TLS settings for connecting to the endpoint for TCP connections. | no       |
+| Name                                                        | Description                                                                 | Required |
+|-------------------------------------------------------------|-----------------------------------------------------------------------------|----------|
+| [`listener`][listener]                                      | Configures a listener for Syslog messages.                                  | no       |
+| `listener` > [`raw_format_options`][raw_format_options]     | Configures `raw` syslog format behavior.                                    | no       |
+| `listener` > [`rfc3164_cisco_components`][cisco_components] | Configures parsing of non-standard Cisco IOS syslog extensions.             | no       |
+| `listener` > [`tls_config`][tls_config]                     | Configures TLS settings for connecting to the endpoint for TCP connections. | no       |
 
 The > symbol indicates deeper levels of nesting.
 For example, `listener` > `tls_config` refers to a `tls_config` block defined inside a `listener` block.
 
 [listener]: #listener
 [tls_config]: #tls_config
 [raw_format_options]: #raw_format_options
+[cisco_components]: #rfc3164_cisco_components
 
 ### `listener`
 
@@ -150,16 +154,16 @@ The `rfc3164_default_to_current_year`, `use_incoming_timestamp` and `use_rfc5424
   Disables log line parsing. This format allows receiving non-RFC5424 compliant logs, such as [CEF][cef].
   Raw logs can be forwarded to [`loki.process`](./loki.process.md) component for parsing.
 
+[cef]: https://www.splunk.com/en_us/blog/learn/common-event-format-cef.html
+
 {{< admonition type="note" >}}
 The `raw` format is an [experimental][] feature.
 Experimental features are subject to frequent breaking changes, and may be removed with no equivalent replacement.
 To enable and use an experimental feature, you must set the `stability.level` [flag][] to `experimental`.
-{{< /admonition >}}
 
 [flag]: https://grafana.com/docs/alloy/<ALLOY_VERSION>/reference/cli/run/
 [experimental]: https://grafana.com/docs/release-life-cycle/
-
-[cef]: https://www.splunk.com/en_us/blog/learn/common-event-format-cef.html
+{{< /admonition >}}
 
 ### `raw_format_options`
 
@@ -177,6 +181,65 @@ The following argument is supported:
 |---------------------------------|--------|-----------------------------------------------------------------------------|---------|----------|
 | `use_null_terminator_delimiter` | `bool` | Use null-terminator (`\0`) instead of line break (`\n`) to split log lines. | `false` | no       |
 
+### `rfc3164_cisco_components`
+
+{{< docs/shared lookup="stability/experimental_feature.md" source="alloy" version="<ALLOY_VERSION>" >}}
+
+The `rfc3164_cisco_components` configures parsing of non-standard Cisco IOS syslog extensions. 
+
+{{< admonition type="note" >}}
+This block can only be used when you set `syslog_format` to `rfc3164`.
+{{< /admonition >}}
+
+The following arguments are supported:
+
+| Name               | Type   | Description                                     | Default | Required |
+|--------------------|--------|-------------------------------------------------|---------|----------|
+| `enable_all`       | `bool` | Enables all components below.                   | `false` | no       |
+| `message_counter`  | `bool` | Enables syslog message counter field parsing.   | `false` | no       |
+| `sequence_number`  | `bool` | Enables service sequence number field parsing.  | `false` | no       |
+| `hostname`         | `bool` | Enables origin hostname field parsing.          | `false` | no       |
+| `second_fractions` | `bool` | Enables milliseconds parsing in timestamp field.| `false` | no       |
+
+{{< admonition type="note" >}}
+At least one option has to be enabled if `enable_all` is set to `false`.
+{{< /admonition >}}
+
+{{< admonition type="caution" >}}
+The `rfc3164_cisco_components` configuration must match your Cisco device configuration. 
+The `loki.source.syslog` component cannot auto-detect which components are present because they share similar formats.
+{{< /admonition >}}
+
+#### Cisco Device Configuration
+
+```
+conf t
+
+! Enable message counter (on by default for remote logging)
+logging host 10.0.0.10
+
+! Add service sequence numbers
+service sequence-numbers
+
+! Add origin hostname
+logging origin-id hostname
+
+! Enable millisecond timestamps
+service timestamps log datetime msec localtime
+
+! Recommended: Enable NTP to remove asterisk
+ntp server <your-ntp-server>
+```
+
+#### Current Limitations
+
+* **Component Ordering**: When Cisco components are selectively disabled on the device but the parser expects them, parsing will fail or produce incorrect results. 
+  Always match your parser configuration to your device configuration.
+* **Structured Data**: Messages with RFC5424-style structured data blocks (from `logging host X session-id` or `sequence-num-session`) are not currently supported.
+  See the [upstream issue][go-syslog-issue] for details.
+
+[go-syslog-issue]: https://github.com/leodido/go-syslog/issues/35
+
 ### `tls_config`
 
 {{< docs/shared lookup="reference/components/tls-config-block.md" source="alloy" version="<ALLOY_VERSION>" >}}

internal/component/loki/source/syslog/config/config.go

@@ -55,6 +55,15 @@ func (s SyslogFormat) Validate() error {
 	return fmt.Errorf("unknown syslog format: %q", s)
 }
 
+// RFC3164CiscoComponents enables Cisco IOS log line parsing and configures what fields to parse.
+type RFC3164CiscoComponents struct {
+	EnableAll       bool
+	MessageCounter  bool
+	SequenceNumber  bool
+	Hostname        bool
+	SecondFractions bool
+}
+
 // RawFormatOptions are options for raw syslog format processing.
 type RawFormatOptions struct {
 	// UseNullTerminatorDelimiter sets null terminator ('\0') as a log line delimiter for non-transparent framed messages.
@@ -116,6 +125,9 @@ type SyslogTargetConfig struct {
 	// When parsing an RFC3164 message, should the year be defaulted to the current year?
 	// When false, the year will default to 0.
 	RFC3164DefaultToCurrentYear bool `yaml:"rfc3164_default_to_current_year"`
+
+	// RFC3164CiscoComponents enables and configures Cisco IOS syslog parsing.
+	RFC3164CiscoComponents *RFC3164CiscoComponents `yaml:"rfc3164_cisco_components"`
 }
 
 func (config SyslogTargetConfig) IsRFC3164Message() bool {

internal/component/loki/source/syslog/internal/syslogtarget/syslogparser/syslogparser.go

@@ -46,11 +46,51 @@ func isDigit(b byte) bool {
 	return b >= '0' && b <= '9'
 }
 
+type RFC3164CiscoComponents struct {
+	MessageCounter  bool
+	SequenceNumber  bool
+	CiscoHostname   bool
+	SecondFractions bool
+}
+
+type StreamParseConfig struct {
+	MaxMessageLength       int
+	IsRFC3164Message       bool
+	UseRFC3164DefaultYear  bool
+	RFC3164CiscoComponents *RFC3164CiscoComponents
+}
+
+func (cfg StreamParseConfig) ciscoComponentOptions() []syslog.MachineOption {
+	cmps := cfg.RFC3164CiscoComponents
+	if cmps == nil {
+		return nil
+	}
+
+	opts := make([]syslog.MachineOption, 0, 4) // max number of cisco components
+	if cmps.MessageCounter {
+		opts = append(opts, rfc3164.WithMessageCounter())
+	}
+
+	if cmps.SequenceNumber {
+		opts = append(opts, rfc3164.WithSequenceNumber())
+	}
+
+	if cmps.CiscoHostname {
+		opts = append(opts, rfc3164.WithCiscoHostname())
+	}
+
+	if cmps.SecondFractions {
+		opts = append(opts, rfc3164.WithSecondFractions())
+	}
+
+	return opts
+}
+
 // ParseStream parses a rfc5424 syslog stream from the given Reader, calling
 // the callback function with the parsed messages. The parser automatically
 // detects octet counting.
 // The function returns on EOF or unrecoverable errors.
-func ParseStream(isRFC3164Message bool, useRFC3164DefaultYear bool, r io.Reader, callback func(res *syslog.Result), maxMessageLength int) error {
+func ParseStream(cfg StreamParseConfig, r io.Reader, callback func(res *syslog.Result)) error {
 	buf := bufio.NewReaderSize(r, 1<<10)
 
 	b, err := buf.ReadByte()
@@ -59,7 +99,7 @@ func ParseStream(isRFC3164Message bool, useRFC3164DefaultYear bool, r io.Reader,
 	}
 	_ = buf.UnreadByte()
 	cb := callback
-	if isRFC3164Message && useRFC3164DefaultYear {
+	if cfg.IsRFC3164Message && cfg.UseRFC3164DefaultYear {
 		cb = func(res *syslog.Result) {
 			if res.Message != nil {
 				rfc3164Msg := res.Message.(*rfc3164.SyslogMessage)
@@ -71,26 +111,39 @@ func ParseStream(isRFC3164Message bool, useRFC3164DefaultYear bool, r io.Reader,
 		}
 	}
 
+	opts := []syslog.ParserOption{
+		syslog.WithListener(cb),
+		syslog.WithMaxMessageLength(cfg.MaxMessageLength),
+		syslog.WithBestEffort(),
+	}
+
+	if cfg.IsRFC3164Message && cfg.RFC3164CiscoComponents != nil {
+		machineOpts := cfg.ciscoComponentOptions()
+		opts = append(opts, syslog.WithMachineOptions(machineOpts...))
+	}
+
 	// See https://datatracker.ietf.org/doc/html/rfc6587 for details on message framing
 	// If a syslog message starts with '<' the first piece of the message is the priority, which means it must use
 	// an explicit framing character.
+	var parserFunc func(args ...syslog.ParserOption) syslog.Parser
 	switch framingTypeFromFirstByte(b) {
 	case framingTypeNonTransparent:
-		if isRFC3164Message {
-			nontransparent.NewParserRFC3164(syslog.WithListener(cb), syslog.WithMaxMessageLength(maxMessageLength), syslog.WithBestEffort()).Parse(buf)
+		if cfg.IsRFC3164Message {
+			parserFunc = nontransparent.NewParserRFC3164
 		} else {
-			nontransparent.NewParser(syslog.WithListener(cb), syslog.WithMaxMessageLength(maxMessageLength), syslog.WithBestEffort()).Parse(buf)
+			parserFunc = nontransparent.NewParser
 		}
 	case framingTypeOctetCounting:
 		// If a syslog message starts with a digit, it must use octet counting, and the first piece of the message is the length
-		if isRFC3164Message {
-			octetcounting.NewParserRFC3164(syslog.WithListener(cb), syslog.WithMaxMessageLength(maxMessageLength), syslog.WithBestEffort()).Parse(buf)
+		if cfg.IsRFC3164Message {
+			parserFunc = octetcounting.NewParserRFC3164
 		} else {
-			octetcounting.NewParser(syslog.WithListener(cb), syslog.WithMaxMessageLength(maxMessageLength), syslog.WithBestEffort()).Parse(buf)
+			parserFunc = octetcounting.NewParser
 		}
 	default:
 		return fmt.Errorf("invalid or unsupported framing. first byte: %q", b)
 	}
 
+	parserFunc(opts...).Parse(buf)
 	return nil
 }

internal/component/loki/source/syslog/internal/syslogtarget/syslogparser/syslogparser_test.go

@@ -6,18 +6,17 @@ import (
 	"testing"
 	"time"
 
-	"github.com/grafana/alloy/internal/component/loki/source/syslog/internal/syslogtarget/syslogparser"
-	"github.com/grafana/alloy/internal/util"
 	"github.com/leodido/go-syslog/v4"
 	"github.com/leodido/go-syslog/v4/rfc3164"
 	"github.com/leodido/go-syslog/v4/rfc5424"
 	"github.com/stretchr/testify/require"
-)
 
-var (
-	defaultMaxMessageLength = 8192
+	"github.com/grafana/alloy/internal/component/loki/source/syslog/internal/syslogtarget/syslogparser"
+	"github.com/grafana/alloy/internal/util"
 )
 
+var defaultMaxMessageLength = 8192
+
 func TestParseStream_OctetCounting(t *testing.T) {
 	r := strings.NewReader("23 <13>1 - - - - - - First24 <13>1 - - - - - - Second")
 
@@ -26,7 +25,7 @@ func TestParseStream_OctetCounting(t *testing.T) {
 		results = append(results, res)
 	}
 
-	err := syslogparser.ParseStream(false, false, r, cb, defaultMaxMessageLength)
+	err := syslogparser.ParseStream(syslogparser.StreamParseConfig{MaxMessageLength: defaultMaxMessageLength}, r, cb)
 	require.NoError(t, err)
 
 	require.Equal(t, 2, len(results))
@@ -45,7 +44,7 @@ func TestParseStream_ValidParseError(t *testing.T) {
 		results = append(results, res)
 	}
 
-	err := syslogparser.ParseStream(false, false, r, cb, defaultMaxMessageLength)
+	err := syslogparser.ParseStream(syslogparser.StreamParseConfig{MaxMessageLength: defaultMaxMessageLength}, r, cb)
 	require.NoError(t, err)
 
 	require.Equal(t, 1, len(results))
@@ -61,7 +60,7 @@ func TestParseStream_OctetCounting_LongMessage(t *testing.T) {
 		results = append(results, res)
 	}
 
-	err := syslogparser.ParseStream(false, false, r, cb, defaultMaxMessageLength)
+	err := syslogparser.ParseStream(syslogparser.StreamParseConfig{MaxMessageLength: defaultMaxMessageLength}, r, cb)
 	require.NoError(t, err)
 
 	require.Equal(t, 1, len(results))
@@ -76,7 +75,7 @@ func TestParseStream_NewlineSeparated(t *testing.T) {
 		results = append(results, res)
 	}
 
-	err := syslogparser.ParseStream(false, false, r, cb, defaultMaxMessageLength)
+	err := syslogparser.ParseStream(syslogparser.StreamParseConfig{MaxMessageLength: defaultMaxMessageLength}, r, cb)
 	require.NoError(t, err)
 
 	require.Equal(t, 2, len(results))
@@ -89,14 +88,14 @@ func TestParseStream_NewlineSeparated(t *testing.T) {
 func TestParseStream_InvalidStream(t *testing.T) {
 	r := strings.NewReader("invalid")
 
-	err := syslogparser.ParseStream(false, false, r, func(_ *syslog.Result) {}, defaultMaxMessageLength)
+	err := syslogparser.ParseStream(syslogparser.StreamParseConfig{MaxMessageLength: defaultMaxMessageLength}, r, func(_ *syslog.Result) {})
 	require.EqualError(t, err, "invalid or unsupported framing. first byte: 'i'")
 }
 
 func TestParseStream_EmptyStream(t *testing.T) {
 	r := strings.NewReader("")
 
-	err := syslogparser.ParseStream(false, false, r, func(_ *syslog.Result) {}, defaultMaxMessageLength)
+	err := syslogparser.ParseStream(syslogparser.StreamParseConfig{MaxMessageLength: defaultMaxMessageLength}, r, func(_ *syslog.Result) {})
 	require.Equal(t, err, io.EOF)
 }
 
@@ -108,7 +107,7 @@ func TestParseStream_RFC3164Timestamp(t *testing.T) {
 		results = append(results, res)
 	}
 
-	err := syslogparser.ParseStream(true, false, r, cb, defaultMaxMessageLength)
+	err := syslogparser.ParseStream(syslogparser.StreamParseConfig{MaxMessageLength: defaultMaxMessageLength, IsRFC3164Message: true}, r, cb)
 	require.NoError(t, err)
 
 	require.Equal(t, 1, len(results))
@@ -126,7 +125,7 @@ func TestParseStream_RFC3164TimestampWithYear(t *testing.T) {
 		results = append(results, res)
 	}
 
-	err := syslogparser.ParseStream(true, true, r, cb, defaultMaxMessageLength)
+	err := syslogparser.ParseStream(syslogparser.StreamParseConfig{MaxMessageLength: defaultMaxMessageLength, IsRFC3164Message: true, UseRFC3164DefaultYear: true}, r, cb)
 	require.NoError(t, err)
 
 	require.Equal(t, 1, len(results))

internal/component/loki/source/syslog/internal/syslogtarget/syslogtarget.go

@@ -8,6 +8,7 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"strconv"
 	"strings"
 	"time"
 
@@ -128,6 +129,14 @@ func (t *SyslogTarget) handleMessageRFC5424(connLabels labels.Labels, msg *rfc54
 		lb.Set("__syslog_message_msg_id", *v)
 	}
 
+	// cisco-specific fields
+	if v := msg.MessageCounter; v != nil {
+		lb.Set("__syslog_message_msg_counter", strconv.Itoa(int(*v)))
+	}
+	if v := msg.Sequence; v != nil {
+		lb.Set("__syslog_message_sequence", strconv.Itoa(int(*v)))
+	}
+
 	if t.config.LabelStructuredData && msg.StructuredData != nil {
 		for id, params := range *msg.StructuredData {
 			id = strings.ReplaceAll(id, "@", "_")
@@ -193,6 +202,14 @@ func (t *SyslogTarget) handleMessageRFC3164(connLabels labels.Labels, msg *rfc31
 		lb.Set("__syslog_message_msg_id", *v)
 	}
 
+	// cisco-specific fields
+	if v := msg.MessageCounter; v != nil {
+		lb.Set("__syslog_message_msg_counter", strconv.Itoa(int(*v)))
+	}
+	if v := msg.Sequence; v != nil {
+		lb.Set("__syslog_message_sequence", strconv.Itoa(int(*v)))
+	}
+
 	processed, _ := relabel.Process(lb.Labels(), t.relabelConfig...)
 
 	filtered := make(model.LabelSet)