From 40badfff740a710f7c747fa99b3aa16f9b0cf077 Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Tue, 15 Jan 2019 12:17:34 +0100 Subject: [PATCH 1/5] Add optional TCP framing to syslog adapter Enable users to specify "octet-counted" frames as described in RFC6587 (Syslog over TCP) 3.4.1 and RFC5424 (Syslog over TLS). This prefixes each message with the length of the message to allow consumers to easily determine where the message ends (rather than traditional LF framing). This also enables multiline Syslog messages without escaping. This keeps the default as LF framing for backwards compatibility though octet-counted framing is preferred when it is known the downstream consumer can handle it. --- adapters/syslog/syslog.go | 44 +++++++++++++++++++++++++++++ adapters/syslog/syslog_test.go | 51 ++++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+) diff --git a/adapters/syslog/syslog.go b/adapters/syslog/syslog.go index ee9839a6..f844bbae 100644 --- a/adapters/syslog/syslog.go +++ b/adapters/syslog/syslog.go @@ -2,6 +2,7 @@ package syslog import ( "bytes" + "crypto/tls" "errors" "fmt" "io/ioutil" @@ -24,9 +25,17 @@ const defaultRetryCount = 10 var ( hostname string retryCount uint + tcpFraming TCPFraming econnResetErrStr string ) +type TCPFraming string + +const ( + TraditionalTCPFraming TCPFraming = "traditional" // LF framing + OctetCountedTCPFraming = "octet-counted" // https://tools.ietf.org/html/rfc6587#section-3.4.1 +) + func init() { hostname, _ = os.Hostname() econnResetErrStr = fmt.Sprintf("write: %s", syscall.ECONNRESET.Error()) @@ -89,6 +98,17 @@ func NewSyslogAdapter(route *router.Route) (router.LogAdapter, error) { structuredData = fmt.Sprintf("[%s]", structuredData) } + if isTCPConnecion(conn) { + switch s := cfg.GetEnvDefault("SYSLOG_TCP_FRAMING", "traditional"); s { + case "traditional": + tcpFraming = TraditionalTCPFraming + case "octet-counted": + tcpFraming = OctetCountedTCPFraming + default: + return nil, fmt.Errorf("unknown SYSLOG_TCP_FRAMING value: %s", s) + } + } + var tmplStr string switch format { case "rfc5424": @@ -137,6 +157,19 @@ func (a *Adapter) Stream(logstream chan *router.Message) { log.Println("syslog:", err) return } + + if isTCPConnecion(a.conn) { + switch tcpFraming { + case OctetCountedTCPFraming: + buf = append([]byte(fmt.Sprintf("%d ", len(buf))), buf...) + case TraditionalTCPFraming: + // leave as-is + default: + // should never get here, validated above + panic("unknown framing format: " + tcpFraming) + } + } + if _, err = a.conn.Write(buf); err != nil { log.Println("syslog:", err) switch a.conn.(type) { @@ -226,6 +259,17 @@ func retryExp(fun func() error, tries uint) error { } } +func isTCPConnecion(conn net.Conn) bool { + switch conn.(type) { + case *net.TCPConn: + return true + case *tls.Conn: + return true + default: + return false + } +} + // Message extends router.Message for the syslog standard type Message struct { *router.Message diff --git a/adapters/syslog/syslog_test.go b/adapters/syslog/syslog_test.go index 1584266b..c032a27f 100644 --- a/adapters/syslog/syslog_test.go +++ b/adapters/syslog/syslog_test.go @@ -2,6 +2,7 @@ package syslog import ( "bufio" + "fmt" "io" "io/ioutil" "log" @@ -40,6 +41,56 @@ var ( badHostnameContent = "hostname\r\n" ) +func TestSyslogOctetFraming(t *testing.T) { + os.Setenv("SYSLOG_TCP_FRAMING", "octet-counted") + defer os.Unsetenv("SYSLOG_TCP_FRAMING") + + done := make(chan string) + addr, sock, srvWG := startServer("tcp", "", done) + defer srvWG.Wait() + defer os.Remove(addr) + defer sock.Close() + + route := &router.Route{Adapter: "syslog+tcp", Address: addr} + adapter, err := NewSyslogAdapter(route) + if err != nil { + t.Fatal(err) + } + defer adapter.(*Adapter).conn.Close() + + stream := make(chan *router.Message) + go adapter.Stream(stream) + + count := 1 + messages := make(chan string, count) + go sendLogstream(stream, messages, adapter, count) + + timeout := time.After(6 * time.Second) + msgnum := 1 + select { + case msg := <-done: + sizeStr := "" + _, err := fmt.Sscan(msg, &sizeStr) + if err != nil { + t.Fatal("unable to scan size from message: ", err) + } + + size, err := strconv.ParseInt(sizeStr, 10, 32) + if err != nil { + t.Fatal("unable to scan size from message: ", err) + } + + expectedOctetFrame := len(sizeStr) + 1 + int(size) + if len(msg) != expectedOctetFrame { + t.Errorf("expected octet frame to be %d. got %d instead for message %s", expectedOctetFrame, size, msg) + } + return + case <-timeout: + t.Fatal("timeout after", msgnum, "messages") + return + } +} + func TestSyslogRetryCount(t *testing.T) { newRetryCount := uint(20) os.Setenv("RETRY_COUNT", strconv.Itoa(int(newRetryCount))) From 5af7c79da2636595a8cb849b550216f3af74d04a Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Tue, 22 Jan 2019 14:54:23 +0100 Subject: [PATCH 2/5] Add comments to exported syslog types --- adapters/syslog/syslog.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/adapters/syslog/syslog.go b/adapters/syslog/syslog.go index f844bbae..55657887 100644 --- a/adapters/syslog/syslog.go +++ b/adapters/syslog/syslog.go @@ -29,11 +29,14 @@ var ( econnResetErrStr string ) +// TCPFraming represents the type of framing to use for syslog messages type TCPFraming string const ( - TraditionalTCPFraming TCPFraming = "traditional" // LF framing - OctetCountedTCPFraming = "octet-counted" // https://tools.ietf.org/html/rfc6587#section-3.4.1 + // TraditionalTCPFraming is the traditional LF framing of syslog messages on the wire + TraditionalTCPFraming TCPFraming = "traditional" + // OctetCountedTCPFraming prepends the size of each message before the message. https://tools.ietf.org/html/rfc6587#section-3.4.1 + OctetCountedTCPFraming = "octet-counted" ) func init() { From b4d20fe43aaddbf3ce58ca08575e9968467aabc4 Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Fri, 22 Mar 2019 10:33:52 +0100 Subject: [PATCH 3/5] Fix type of OctetCountedTCPFraming to be TCPFraming --- adapters/syslog/syslog.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adapters/syslog/syslog.go b/adapters/syslog/syslog.go index 55657887..580d7108 100644 --- a/adapters/syslog/syslog.go +++ b/adapters/syslog/syslog.go @@ -36,7 +36,7 @@ const ( // TraditionalTCPFraming is the traditional LF framing of syslog messages on the wire TraditionalTCPFraming TCPFraming = "traditional" // OctetCountedTCPFraming prepends the size of each message before the message. https://tools.ietf.org/html/rfc6587#section-3.4.1 - OctetCountedTCPFraming = "octet-counted" + OctetCountedTCPFraming TCPFraming = "octet-counted" ) func init() { From 58ed4309cc2dbc64061014e87fa01cf44b1af2f3 Mon Sep 17 00:00:00 2001 From: Michael Hobbs Date: Fri, 1 May 2020 13:41:07 -0700 Subject: [PATCH 4/5] fix gocyclo complexity --- adapters/syslog/syslog.go | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/adapters/syslog/syslog.go b/adapters/syslog/syslog.go index 580d7108..1454022a 100644 --- a/adapters/syslog/syslog.go +++ b/adapters/syslog/syslog.go @@ -20,7 +20,14 @@ import ( "github.com/gliderlabs/logspout/router" ) -const defaultRetryCount = 10 +const ( + // TraditionalTCPFraming is the traditional LF framing of syslog messages on the wire + TraditionalTCPFraming TCPFraming = "traditional" + // OctetCountedTCPFraming prepends the size of each message before the message. https://tools.ietf.org/html/rfc6587#section-3.4.1 + OctetCountedTCPFraming TCPFraming = "octet-counted" + + defaultRetryCount = 10 +) var ( hostname string @@ -32,13 +39,6 @@ var ( // TCPFraming represents the type of framing to use for syslog messages type TCPFraming string -const ( - // TraditionalTCPFraming is the traditional LF framing of syslog messages on the wire - TraditionalTCPFraming TCPFraming = "traditional" - // OctetCountedTCPFraming prepends the size of each message before the message. https://tools.ietf.org/html/rfc6587#section-3.4.1 - OctetCountedTCPFraming TCPFraming = "octet-counted" -) - func init() { hostname, _ = os.Hostname() econnResetErrStr = fmt.Sprintf("write: %s", syscall.ECONNRESET.Error()) @@ -102,13 +102,8 @@ func NewSyslogAdapter(route *router.Route) (router.LogAdapter, error) { } if isTCPConnecion(conn) { - switch s := cfg.GetEnvDefault("SYSLOG_TCP_FRAMING", "traditional"); s { - case "traditional": - tcpFraming = TraditionalTCPFraming - case "octet-counted": - tcpFraming = OctetCountedTCPFraming - default: - return nil, fmt.Errorf("unknown SYSLOG_TCP_FRAMING value: %s", s) + if err = setTCPFraming(); err != nil { + return nil, err } } @@ -143,6 +138,19 @@ func NewSyslogAdapter(route *router.Route) (router.LogAdapter, error) { }, nil } +func setTCPFraming() error { + switch s := cfg.GetEnvDefault("SYSLOG_TCP_FRAMING", "traditional"); s { + case "traditional": + tcpFraming = TraditionalTCPFraming + return nil + case "octet-counted": + tcpFraming = OctetCountedTCPFraming + return nil + default: + return fmt.Errorf("unknown SYSLOG_TCP_FRAMING value: %s", s) + } +} + // Adapter streams log output to a connection in the Syslog format type Adapter struct { conn net.Conn From d4d807127fe8701a758c1203fd6683cb7962445c Mon Sep 17 00:00:00 2001 From: Michael Hobbs Date: Fri, 1 May 2020 13:49:41 -0700 Subject: [PATCH 5/5] release 3.2.10 --- CHANGELOG.md | 10 +++++++++- VERSION | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cfbf58a2..3e74b5e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,13 @@ All notable changes to this project will be documented in this file. ### Changed +## [v3.2.10] - 2020-05-1 +### Added +- @jszwedko Add optional TCP framing to syslog adapter + +### Fixed +- @bbigras add missing syntax highlighting in README.md + ## [v3.2.9] - 2020-04-30 ### Fixed - @bbigras add missing syntax highlighting in README.md @@ -227,7 +234,8 @@ All notable changes to this project will be documented in this file. - Base container is now Alpine - Moved to gliderlabs organization -[unreleased]: https://github.com/gliderlabs/logspout/compare/v3.2.9...HEAD +[unreleased]: https://github.com/gliderlabs/logspout/compare/v3.2.10...HEAD +[v3.2.10]: https://github.com/gliderlabs/logspout/compare/v3.2.9...v3.2.10 [v3.2.9]: https://github.com/gliderlabs/logspout/compare/v3.2.8...v3.2.9 [v3.2.8]: https://github.com/gliderlabs/logspout/compare/v3.2.7...v3.2.8 [v3.2.7]: https://github.com/gliderlabs/logspout/compare/v3.2.6...v3.2.7 diff --git a/VERSION b/VERSION index eb2a887f..9e90abd9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v3.2.9 +v3.2.10