From 35e39c6cf9bbaa0e3059b00b28fe1d72e33aeda8 Mon Sep 17 00:00:00 2001 From: Nazar Date: Thu, 16 Mar 2023 20:15:21 +0200 Subject: [PATCH 1/2] Add ability to cut postfix with regexp pattern from the containers log file name --- app/discovery/events.go | 46 +++++++++++++++++++++++++---------- app/discovery/events_test.go | 27 ++++++++++++++------- app/main.go | 34 +++++++++++++++++--------- app/main_test.go | 47 ++++++++++++++++++++++++++++++++---- 4 files changed, 116 insertions(+), 38 deletions(-) diff --git a/app/discovery/events.go b/app/discovery/events.go index 83cb66e..a05c186 100644 --- a/app/discovery/events.go +++ b/app/discovery/events.go @@ -12,12 +12,13 @@ import ( // EventNotif emits all changes from all containers states type EventNotif struct { - dockerClient DockerClient - excludes []string - includes []string - includesRegexp *regexp.Regexp - excludesRegexp *regexp.Regexp - eventsCh chan Event + dockerClient DockerClient + excludes []string + includes []string + includesRegexp *regexp.Regexp + excludesRegexp *regexp.Regexp + postfixPatternRegexp *regexp.Regexp + eventsCh chan Event } // Event is simplified docker.APIEvents for containers only, exposed to caller @@ -27,6 +28,7 @@ type Event struct { Group string // group is the "path" part of the image tag, i.e. for umputun/system/logger:latest it will be "system" TS time.Time Status bool + FileName string } // DockerClient defines interface listing containers and subscribing to events @@ -38,7 +40,7 @@ type DockerClient interface { var reGroup = regexp.MustCompile(`/(.*?)/`) // NewEventNotif makes EventNotif publishing all changes to eventsCh -func NewEventNotif(dockerClient DockerClient, excludes, includes []string, includesPattern, excludesPattern string) (*EventNotif, error) { +func NewEventNotif(dockerClient DockerClient, excludes, includes []string, includesPattern, excludesPattern, postfixPattern string) (*EventNotif, error) { log.Printf("[DEBUG] create events notif, excludes: %+v, includes: %+v, includesPattern: %+v, excludesPattern: %+v", excludes, includes, includesPattern, excludesPattern) @@ -59,13 +61,22 @@ func NewEventNotif(dockerClient DockerClient, excludes, includes []string, inclu } } + var postfixPatternRe *regexp.Regexp + if postfixPattern != "" { + postfixPatternRe, err = regexp.Compile(postfixPattern) + if err != nil { + return nil, errors.Wrap(err, "failed to compile postfix number regexp") + } + } + res := EventNotif{ - dockerClient: dockerClient, - excludes: excludes, - includes: includes, - includesRegexp: includesRe, - excludesRegexp: excludesRe, - eventsCh: make(chan Event, 100), + dockerClient: dockerClient, + excludes: excludes, + includes: includes, + includesRegexp: includesRe, + excludesRegexp: excludesRe, + postfixPatternRegexp: postfixPatternRe, + eventsCh: make(chan Event, 100), } // first get all currently running containers @@ -119,6 +130,7 @@ func (e *EventNotif) activate(client DockerClient) { Status: contains(dockerEvent.Status, upStatuses), TS: time.Unix(dockerEvent.Time/1000, dockerEvent.TimeNano), Group: e.group(dockerEvent.From), + FileName: e.formatFileName(containerName), } log.Printf("[INFO] new event %+v", event) e.eventsCh <- event @@ -146,6 +158,7 @@ func (e *EventNotif) emitRunningContainers() error { ContainerID: c.ID, TS: time.Unix(c.Created/1000, 0), Group: e.group(c.Image), + FileName: e.formatFileName(containerName), } log.Printf("[DEBUG] running container added, %+v", event) e.eventsCh <- event @@ -179,6 +192,13 @@ func (e *EventNotif) isAllowed(containerName string) bool { return true } +func (e *EventNotif) formatFileName(containerName string) string { + if e.postfixPatternRegexp != nil { + return e.postfixPatternRegexp.ReplaceAllString(containerName, "") + } + return containerName +} + func contains(e string, s []string) bool { for _, a := range s { if a == e { diff --git a/app/discovery/events_test.go b/app/discovery/events_test.go index 9e3def7..b8d39ae 100644 --- a/app/discovery/events_test.go +++ b/app/discovery/events_test.go @@ -13,7 +13,7 @@ import ( func TestEvents(t *testing.T) { client := &mockDockerClient{} - events, err := NewEventNotif(client, []string{"tst_exclude"}, []string{}, "", "") + events, err := NewEventNotif(client, []string{"tst_exclude"}, []string{}, "", "", "") require.NoError(t, err) time.Sleep(10 * time.Millisecond) go client.add("id1", "name1") @@ -30,7 +30,7 @@ func TestEvents(t *testing.T) { func TestEventsIncludes(t *testing.T) { client := &mockDockerClient{} - events, err := NewEventNotif(client, []string{}, []string{"tst_included"}, "", "") + events, err := NewEventNotif(client, []string{}, []string{"tst_included"}, "", "", "") require.NoError(t, err) time.Sleep(10 * time.Millisecond) go client.add("id2", "tst_included") @@ -53,7 +53,7 @@ func TestEmit(t *testing.T) { client.add("id2", "tst_exclude") client.add("id2", "name2") - events, err := NewEventNotif(client, []string{"tst_exclude"}, []string{}, "", "") + events, err := NewEventNotif(client, []string{"tst_exclude"}, []string{}, "", "", "") require.NoError(t, err) ev := <-events.Channel() @@ -73,7 +73,7 @@ func TestEmitIncludes(t *testing.T) { client.add("id2", "tst_include") client.add("id2", "name2") - events, err := NewEventNotif(client, []string{}, []string{"tst_include"}, "", "") + events, err := NewEventNotif(client, []string{}, []string{"tst_include"}, "", "", "") require.NoError(t, err) ev := <-events.Channel() @@ -84,13 +84,13 @@ func TestEmitIncludes(t *testing.T) { func TestNewEventNotifWithNils(t *testing.T) { client := &mockDockerClient{} - _, err := NewEventNotif(client, nil, nil, "", "") + _, err := NewEventNotif(client, nil, nil, "", "", "") require.NoError(t, err) } func TestIsAllowedExclude(t *testing.T) { client := &mockDockerClient{} - events, err := NewEventNotif(client, []string{"tst_exclude"}, nil, "", "") + events, err := NewEventNotif(client, []string{"tst_exclude"}, nil, "", "", "") require.NoError(t, err) assert.True(t, events.isAllowed("name1")) @@ -99,7 +99,7 @@ func TestIsAllowedExclude(t *testing.T) { func TestIsAllowedExcludePattern(t *testing.T) { client := &mockDockerClient{} - events, err := NewEventNotif(client, nil, nil, "", "tst_exclude.*") + events, err := NewEventNotif(client, nil, nil, "", "tst_exclude.*", "") require.NoError(t, err) assert.True(t, events.isAllowed("tst_include")) @@ -110,7 +110,7 @@ func TestIsAllowedExcludePattern(t *testing.T) { func TestIsAllowedInclude(t *testing.T) { client := &mockDockerClient{} - events, err := NewEventNotif(client, nil, []string{"tst_include"}, "", "") + events, err := NewEventNotif(client, nil, []string{"tst_include"}, "", "", "") require.NoError(t, err) assert.True(t, events.isAllowed("tst_include")) @@ -120,7 +120,7 @@ func TestIsAllowedInclude(t *testing.T) { func TestIsAllowedIncludePattern(t *testing.T) { client := &mockDockerClient{} - events, err := NewEventNotif(client, nil, nil, "tst_include.*", "") + events, err := NewEventNotif(client, nil, nil, "tst_include.*", "", "") require.NoError(t, err) assert.True(t, events.isAllowed("tst_include")) @@ -129,6 +129,15 @@ func TestIsAllowedIncludePattern(t *testing.T) { assert.False(t, events.isAllowed("tst_exclude_no")) } +func TestFormatFileName(t *testing.T) { + client := &mockDockerClient{} + events, err := NewEventNotif(client, nil, nil, "", "", "(_)\\d+$") + require.NoError(t, err) + + assert.Equal(t, "test", events.formatFileName("test")) + assert.Equal(t, "test", events.formatFileName("test_123")) +} + func TestGroup(t *testing.T) { d := EventNotif{} tbl := []struct { diff --git a/app/main.go b/app/main.go index bdf72fa..807a9af 100644 --- a/app/main.go +++ b/app/main.go @@ -28,12 +28,13 @@ type cliOpts struct { SyslogHost string `long:"syslog-host" env:"SYSLOG_HOST" default:"127.0.0.1:514" description:"syslog host"` SyslogPrefix string `long:"syslog-prefix" env:"SYSLOG_PREFIX" default:"docker/" description:"syslog prefix"` - EnableFiles bool `long:"files" env:"LOG_FILES" description:"enable logging to files"` - MaxFileSize int `long:"max-size" env:"MAX_SIZE" default:"10" description:"size of log triggering rotation (MB)"` - MaxFilesCount int `long:"max-files" env:"MAX_FILES" default:"5" description:"number of rotated files to retain"` - MaxFilesAge int `long:"max-age" env:"MAX_AGE" default:"30" description:"maximum number of days to retain"` - MixErr bool `long:"mix-err" env:"MIX_ERR" description:"send error to std output log file"` - FilesLocation string `long:"loc" env:"LOG_FILES_LOC" default:"logs" description:"log files locations"` + EnableFiles bool `long:"files" env:"LOG_FILES" description:"enable logging to files"` + MaxFileSize int `long:"max-size" env:"MAX_SIZE" default:"10" description:"size of log triggering rotation (MB)"` + MaxFilesCount int `long:"max-files" env:"MAX_FILES" default:"5" description:"number of rotated files to retain"` + MaxFilesAge int `long:"max-age" env:"MAX_AGE" default:"30" description:"maximum number of days to retain"` + MixErr bool `long:"mix-err" env:"MIX_ERR" description:"send error to std output log file"` + FilesLocation string `long:"loc" env:"LOG_FILES_LOC" default:"logs" description:"log files locations"` + FilePostfixExcludePattern string `long:"exclude-postfix" env:"FILE_POSTFIX_EXCLUDE_PATTERN" default:"" description:"exclude postfix from log file names with regexp pattern"` //nolint:lll Excludes []string `short:"x" long:"exclude" env:"EXCLUDE" env-delim:"," description:"excluded container names"` Includes []string `short:"i" long:"include" env:"INCLUDE" env-delim:"," description:"included container names"` @@ -96,6 +97,17 @@ func do(ctx context.Context, opts *cliOpts) error { } } + if opts.FilePostfixExcludePattern != "" { + if opts.EnableFiles { + _, err := regexp.Compile(opts.FilePostfixExcludePattern) + if err != nil { + return errors.New("could not parse FilePostfixExcludePattern") + } + } else { + return errors.New("FilePostfixExcludePattern is only allowed when logging to files") + } + } + if opts.EnableSyslog && !syslog.IsSupported() { return errors.New("syslog is not supported on this OS") } @@ -105,7 +117,7 @@ func do(ctx context.Context, opts *cliOpts) error { return errors.Wrapf(err, "failed to make docker client %s", err) } - events, err := discovery.NewEventNotif(client, opts.Excludes, opts.Includes, opts.IncludesPattern, opts.ExcludesPattern) + events, err := discovery.NewEventNotif(client, opts.Excludes, opts.Includes, opts.IncludesPattern, opts.ExcludesPattern, opts.FilePostfixExcludePattern) if err != nil { return errors.Wrap(err, "failed to make event notifier") } @@ -127,7 +139,7 @@ func runEventLoop(ctx context.Context, opts *cliOpts, events *discovery.EventNot return } - logWriter, errWriter := makeLogWriters(opts, event.ContainerName, event.Group) + logWriter, errWriter := makeLogWriters(opts, event.ContainerName, event.FileName, event.Group) ls := logger.LogStreamer{ DockerClient: client, ContainerID: event.ContainerID, @@ -183,7 +195,7 @@ func runEventLoop(ctx context.Context, opts *cliOpts, events *discovery.EventNot // makeLogWriters creates io.Writer with rotated out and separate err files. Also adds writer for remote syslog // //nolint:funlen -func makeLogWriters(opts *cliOpts, containerName, group string) (logWriter, errWriter io.WriteCloser) { +func makeLogWriters(opts *cliOpts, containerName, fileName, group string) (logWriter, errWriter io.WriteCloser) { log.Printf("[DEBUG] create log writer for %s", strings.TrimPrefix(group+"/"+containerName, "/")) if !opts.EnableFiles && !opts.EnableSyslog { log.Fatalf("[ERROR] either files or syslog has to be enabled") @@ -201,7 +213,7 @@ func makeLogWriters(opts *cliOpts, containerName, group string) (logWriter, errW log.Fatalf("[ERROR] can't make directory %s, %v", logDir, err) } - logName := fmt.Sprintf("%s/%s.log", logDir, containerName) + logName := fmt.Sprintf("%s/%s.log", logDir, fileName) logFileWriter := &lumberjack.Logger{ Filename: logName, MaxSize: opts.MaxFileSize, // megabytes @@ -215,7 +227,7 @@ func makeLogWriters(opts *cliOpts, containerName, group string) (logWriter, errW errFname := logName if !opts.MixErr { // if writers not mixed make error writer - errFname = fmt.Sprintf("%s/%s.err", logDir, containerName) + errFname = fmt.Sprintf("%s/%s.err", logDir, fileName) errFileWriter = &lumberjack.Logger{ Filename: errFname, MaxSize: opts.MaxFileSize, // megabytes diff --git a/app/main_test.go b/app/main_test.go index 0cb9758..1d3d30e 100644 --- a/app/main_test.go +++ b/app/main_test.go @@ -36,7 +36,7 @@ func Test_makeLogWriters(t *testing.T) { setupLog(true) opts := cliOpts{FilesLocation: "/tmp/logger.test", EnableFiles: true, MaxFileSize: 1, MaxFilesCount: 10} - stdWr, errWr := makeLogWriters(&opts, "container1", "gr1") + stdWr, errWr := makeLogWriters(&opts, "container1", "container1", "gr1") assert.NotEqual(t, stdWr, errWr, "different writers for out and err") // write to out writer @@ -63,12 +63,49 @@ func Test_makeLogWriters(t *testing.T) { assert.NoError(t, errWr.Close()) } +func Test_makeLogWritersToTheSameFile(t *testing.T) { + defer os.RemoveAll("/tmp/logger.test") // nolint + setupLog(true) + + // test with excluded postfix number after underscore + opts := cliOpts{FilesLocation: "/tmp/logger.test", EnableFiles: true, MaxFileSize: 1, MaxFilesCount: 10} + stdWr, errWr := makeLogWriters(&opts, "container1_1", "container1", "gr1") + assert.NotEqual(t, stdWr, errWr, "different writers for out and err") + stdWr2, errWr2 := makeLogWriters(&opts, "container1_2", "container1", "gr1") + assert.NotEqual(t, stdWr2, errWr2, "different writers for out and err") + + // write to out writers + _, err := stdWr.Write([]byte("abc line 1 from container 1\n")) + assert.NoError(t, err) + _, err = stdWr2.Write([]byte("abc line 2 from container 2\n")) + assert.NoError(t, err) + + // write to err writers + _, err = errWr.Write([]byte("err line 1 from container 1\n")) + assert.NoError(t, err) + _, err = errWr2.Write([]byte("err line 2 from container 2\n")) + assert.NoError(t, err) + + r, err := os.ReadFile("/tmp/logger.test/gr1/container1.log") + assert.NoError(t, err) + assert.Equal(t, "abc line 1 from container 1\nabc line 2 from container 2\n", string(r)) + + r, err = os.ReadFile("/tmp/logger.test/gr1/container1.err") + assert.NoError(t, err) + assert.Equal(t, "err line 1 from container 1\nerr line 2 from container 2\n", string(r)) + + assert.NoError(t, stdWr.Close()) + assert.NoError(t, stdWr2.Close()) + assert.NoError(t, errWr.Close()) + assert.NoError(t, errWr2.Close()) +} + func Test_makeLogWritersMixed(t *testing.T) { defer os.RemoveAll("/tmp/logger.test") // nolint setupLog(false) opts := cliOpts{FilesLocation: "/tmp/logger.test", EnableFiles: true, MaxFileSize: 1, MaxFilesCount: 10, MixErr: true} - stdWr, errWr := makeLogWriters(&opts, "container1", "gr1") + stdWr, errWr := makeLogWriters(&opts, "container1", "container1", "gr1") assert.Equal(t, stdWr, errWr, "same writer for out and err in mixed mode") // write to out writer @@ -94,7 +131,7 @@ func Test_makeLogWritersMixed(t *testing.T) { func Test_makeLogWritersWithJSON(t *testing.T) { defer os.RemoveAll("/tmp/logger.test") // nolint opts := cliOpts{FilesLocation: "/tmp/logger.test", EnableFiles: true, MaxFileSize: 1, MaxFilesCount: 10, ExtJSON: true} - stdWr, errWr := makeLogWriters(&opts, "container1", "gr1") + stdWr, errWr := makeLogWriters(&opts, "container1", "container1", "gr1") // write to out writer _, err := stdWr.Write([]byte("abc line 1")) @@ -113,7 +150,7 @@ func Test_makeLogWritersWithJSON(t *testing.T) { func Test_makeLogWritersSyslogFailed(t *testing.T) { opts := cliOpts{EnableSyslog: true} - stdWr, errWr := makeLogWriters(&opts, "container1", "gr1") + stdWr, errWr := makeLogWriters(&opts, "container1", "container1", "gr1") assert.Equal(t, stdWr, errWr, "same writer for out and err in syslog") // write to out writer _, err := stdWr.Write([]byte("abc line 1\n")) @@ -130,7 +167,7 @@ func Test_makeLogWritersSyslogFailed(t *testing.T) { func Test_makeLogWritersSyslogPassed(t *testing.T) { opts := cliOpts{EnableSyslog: true, SyslogHost: "127.0.0.1:514", SyslogPrefix: "docker/"} - stdWr, errWr := makeLogWriters(&opts, "container1", "gr1") + stdWr, errWr := makeLogWriters(&opts, "container1", "container1", "gr1") assert.Equal(t, stdWr, errWr, "same writer for out and err in syslog") // write to out writer From 93123f94e03674f081b0efc20c9b69e9f849945b Mon Sep 17 00:00:00 2001 From: Nazar Date: Mon, 20 Mar 2023 22:11:09 +0200 Subject: [PATCH 2/2] fix formatting errors --- app/discovery/events.go | 3 ++- app/discovery/events_test.go | 2 +- app/main.go | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/app/discovery/events.go b/app/discovery/events.go index a05c186..2a92d15 100644 --- a/app/discovery/events.go +++ b/app/discovery/events.go @@ -40,7 +40,8 @@ type DockerClient interface { var reGroup = regexp.MustCompile(`/(.*?)/`) // NewEventNotif makes EventNotif publishing all changes to eventsCh -func NewEventNotif(dockerClient DockerClient, excludes, includes []string, includesPattern, excludesPattern, postfixPattern string) (*EventNotif, error) { +func NewEventNotif(dockerClient DockerClient, excludes, includes []string, includesPattern, excludesPattern, + postfixPattern string) (*EventNotif, error) { log.Printf("[DEBUG] create events notif, excludes: %+v, includes: %+v, includesPattern: %+v, excludesPattern: %+v", excludes, includes, includesPattern, excludesPattern) diff --git a/app/discovery/events_test.go b/app/discovery/events_test.go index b8d39ae..b938c3f 100644 --- a/app/discovery/events_test.go +++ b/app/discovery/events_test.go @@ -210,7 +210,7 @@ func (m *mockDockerClient) remove(id string) { log.Printf("removed %s", id) } -func (m *mockDockerClient) ListContainers(opts dockerclient.ListContainersOptions) ([]dockerclient.APIContainers, error) { +func (m *mockDockerClient) ListContainers(_ dockerclient.ListContainersOptions) ([]dockerclient.APIContainers, error) { m.Lock() defer m.Unlock() return m.containers, nil diff --git a/app/main.go b/app/main.go index 807a9af..4897b8c 100644 --- a/app/main.go +++ b/app/main.go @@ -117,7 +117,8 @@ func do(ctx context.Context, opts *cliOpts) error { return errors.Wrapf(err, "failed to make docker client %s", err) } - events, err := discovery.NewEventNotif(client, opts.Excludes, opts.Includes, opts.IncludesPattern, opts.ExcludesPattern, opts.FilePostfixExcludePattern) + events, err := discovery.NewEventNotif(client, opts.Excludes, opts.Includes, opts.IncludesPattern, opts.ExcludesPattern, + opts.FilePostfixExcludePattern) if err != nil { return errors.Wrap(err, "failed to make event notifier") }