Skip to content

Add ability to exclude postfix from log file names with regexp pattern #29

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Add ability to exclude postfix from log file names with regexp pattern #29

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
35e39c6
Add ability to cut postfix with regexp pattern from the containers lo…
NoHow Mar 16, 2023
93123f9
fix formatting errors
NoHow Mar 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 34 additions & 13 deletions app/discovery/events.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,13 @@ import (

// EventNotif emits all changes from all containers states
type EventNotif struct {
dockerClient DockerClient
excludes []string
includes []string
includesRegexp *regexp.Regexp
excludesRegexp *regexp.Regexp
eventsCh chan Event
dockerClient DockerClient
excludes []string
includes []string
includesRegexp *regexp.Regexp
excludesRegexp *regexp.Regexp
postfixPatternRegexp *regexp.Regexp
eventsCh chan Event
}

// Event is simplified docker.APIEvents for containers only, exposed to caller
Expand All @@ -27,6 +28,7 @@ type Event struct {
Group string // group is the "path" part of the image tag, i.e. for umputun/system/logger:latest it will be "system"
TS time.Time
Status bool
FileName string
}

// DockerClient defines interface listing containers and subscribing to events
Expand All @@ -38,7 +40,8 @@ type DockerClient interface {
var reGroup = regexp.MustCompile(`/(.*?)/`)

// NewEventNotif makes EventNotif publishing all changes to eventsCh
func NewEventNotif(dockerClient DockerClient, excludes, includes []string, includesPattern, excludesPattern string) (*EventNotif, error) {
func NewEventNotif(dockerClient DockerClient, excludes, includes []string, includesPattern, excludesPattern,
postfixPattern string) (*EventNotif, error) {
log.Printf("[DEBUG] create events notif, excludes: %+v, includes: %+v, includesPattern: %+v, excludesPattern: %+v",
excludes, includes, includesPattern, excludesPattern)

Expand All @@ -59,13 +62,22 @@ func NewEventNotif(dockerClient DockerClient, excludes, includes []string, inclu
}
}

var postfixPatternRe *regexp.Regexp
if postfixPattern != "" {
postfixPatternRe, err = regexp.Compile(postfixPattern)
if err != nil {
return nil, errors.Wrap(err, "failed to compile postfix number regexp")
}
}

res := EventNotif{
dockerClient: dockerClient,
excludes: excludes,
includes: includes,
includesRegexp: includesRe,
excludesRegexp: excludesRe,
eventsCh: make(chan Event, 100),
dockerClient: dockerClient,
excludes: excludes,
includes: includes,
includesRegexp: includesRe,
excludesRegexp: excludesRe,
postfixPatternRegexp: postfixPatternRe,
eventsCh: make(chan Event, 100),
}

// first get all currently running containers
Expand Down Expand Up @@ -119,6 +131,7 @@ func (e *EventNotif) activate(client DockerClient) {
Status: contains(dockerEvent.Status, upStatuses),
TS: time.Unix(dockerEvent.Time/1000, dockerEvent.TimeNano),
Group: e.group(dockerEvent.From),
FileName: e.formatFileName(containerName),
}
log.Printf("[INFO] new event %+v", event)
e.eventsCh <- event
Expand Down Expand Up @@ -146,6 +159,7 @@ func (e *EventNotif) emitRunningContainers() error {
ContainerID: c.ID,
TS: time.Unix(c.Created/1000, 0),
Group: e.group(c.Image),
FileName: e.formatFileName(containerName),
}
log.Printf("[DEBUG] running container added, %+v", event)
e.eventsCh <- event
Expand Down Expand Up @@ -179,6 +193,13 @@ func (e *EventNotif) isAllowed(containerName string) bool {
return true
}

func (e *EventNotif) formatFileName(containerName string) string {
if e.postfixPatternRegexp != nil {
return e.postfixPatternRegexp.ReplaceAllString(containerName, "")
}
return containerName
}

func contains(e string, s []string) bool {
for _, a := range s {
if a == e {
Expand Down
29 changes: 19 additions & 10 deletions app/discovery/events_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import (

func TestEvents(t *testing.T) {
client := &mockDockerClient{}
events, err := NewEventNotif(client, []string{"tst_exclude"}, []string{}, "", "")
events, err := NewEventNotif(client, []string{"tst_exclude"}, []string{}, "", "", "")
require.NoError(t, err)
time.Sleep(10 * time.Millisecond)
go client.add("id1", "name1")
Expand All @@ -30,7 +30,7 @@ func TestEvents(t *testing.T) {

func TestEventsIncludes(t *testing.T) {
client := &mockDockerClient{}
events, err := NewEventNotif(client, []string{}, []string{"tst_included"}, "", "")
events, err := NewEventNotif(client, []string{}, []string{"tst_included"}, "", "", "")
require.NoError(t, err)
time.Sleep(10 * time.Millisecond)
go client.add("id2", "tst_included")
Expand All @@ -53,7 +53,7 @@ func TestEmit(t *testing.T) {
client.add("id2", "tst_exclude")
client.add("id2", "name2")

events, err := NewEventNotif(client, []string{"tst_exclude"}, []string{}, "", "")
events, err := NewEventNotif(client, []string{"tst_exclude"}, []string{}, "", "", "")
require.NoError(t, err)

ev := <-events.Channel()
Expand All @@ -73,7 +73,7 @@ func TestEmitIncludes(t *testing.T) {
client.add("id2", "tst_include")
client.add("id2", "name2")

events, err := NewEventNotif(client, []string{}, []string{"tst_include"}, "", "")
events, err := NewEventNotif(client, []string{}, []string{"tst_include"}, "", "", "")
require.NoError(t, err)

ev := <-events.Channel()
Expand All @@ -84,13 +84,13 @@ func TestEmitIncludes(t *testing.T) {
func TestNewEventNotifWithNils(t *testing.T) {
client := &mockDockerClient{}

_, err := NewEventNotif(client, nil, nil, "", "")
_, err := NewEventNotif(client, nil, nil, "", "", "")
require.NoError(t, err)
}

func TestIsAllowedExclude(t *testing.T) {
client := &mockDockerClient{}
events, err := NewEventNotif(client, []string{"tst_exclude"}, nil, "", "")
events, err := NewEventNotif(client, []string{"tst_exclude"}, nil, "", "", "")
require.NoError(t, err)

assert.True(t, events.isAllowed("name1"))
Expand All @@ -99,7 +99,7 @@ func TestIsAllowedExclude(t *testing.T) {

func TestIsAllowedExcludePattern(t *testing.T) {
client := &mockDockerClient{}
events, err := NewEventNotif(client, nil, nil, "", "tst_exclude.*")
events, err := NewEventNotif(client, nil, nil, "", "tst_exclude.*", "")
require.NoError(t, err)

assert.True(t, events.isAllowed("tst_include"))
Expand All @@ -110,7 +110,7 @@ func TestIsAllowedExcludePattern(t *testing.T) {

func TestIsAllowedInclude(t *testing.T) {
client := &mockDockerClient{}
events, err := NewEventNotif(client, nil, []string{"tst_include"}, "", "")
events, err := NewEventNotif(client, nil, []string{"tst_include"}, "", "", "")
require.NoError(t, err)

assert.True(t, events.isAllowed("tst_include"))
Expand All @@ -120,7 +120,7 @@ func TestIsAllowedInclude(t *testing.T) {

func TestIsAllowedIncludePattern(t *testing.T) {
client := &mockDockerClient{}
events, err := NewEventNotif(client, nil, nil, "tst_include.*", "")
events, err := NewEventNotif(client, nil, nil, "tst_include.*", "", "")
require.NoError(t, err)

assert.True(t, events.isAllowed("tst_include"))
Expand All @@ -129,6 +129,15 @@ func TestIsAllowedIncludePattern(t *testing.T) {
assert.False(t, events.isAllowed("tst_exclude_no"))
}

func TestFormatFileName(t *testing.T) {
client := &mockDockerClient{}
events, err := NewEventNotif(client, nil, nil, "", "", "(_)\\d+$")
require.NoError(t, err)

assert.Equal(t, "test", events.formatFileName("test"))
assert.Equal(t, "test", events.formatFileName("test_123"))
}

func TestGroup(t *testing.T) {
d := EventNotif{}
tbl := []struct {
Expand Down Expand Up @@ -201,7 +210,7 @@ func (m *mockDockerClient) remove(id string) {
log.Printf("removed %s", id)
}

func (m *mockDockerClient) ListContainers(opts dockerclient.ListContainersOptions) ([]dockerclient.APIContainers, error) {
func (m *mockDockerClient) ListContainers(_ dockerclient.ListContainersOptions) ([]dockerclient.APIContainers, error) {
m.Lock()
defer m.Unlock()
return m.containers, nil
Expand Down
35 changes: 24 additions & 11 deletions app/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,13 @@ type cliOpts struct {
SyslogHost string `long:"syslog-host" env:"SYSLOG_HOST" default:"127.0.0.1:514" description:"syslog host"`
SyslogPrefix string `long:"syslog-prefix" env:"SYSLOG_PREFIX" default:"docker/" description:"syslog prefix"`

EnableFiles bool `long:"files" env:"LOG_FILES" description:"enable logging to files"`
MaxFileSize int `long:"max-size" env:"MAX_SIZE" default:"10" description:"size of log triggering rotation (MB)"`
MaxFilesCount int `long:"max-files" env:"MAX_FILES" default:"5" description:"number of rotated files to retain"`
MaxFilesAge int `long:"max-age" env:"MAX_AGE" default:"30" description:"maximum number of days to retain"`
MixErr bool `long:"mix-err" env:"MIX_ERR" description:"send error to std output log file"`
FilesLocation string `long:"loc" env:"LOG_FILES_LOC" default:"logs" description:"log files locations"`
EnableFiles bool `long:"files" env:"LOG_FILES" description:"enable logging to files"`
MaxFileSize int `long:"max-size" env:"MAX_SIZE" default:"10" description:"size of log triggering rotation (MB)"`
MaxFilesCount int `long:"max-files" env:"MAX_FILES" default:"5" description:"number of rotated files to retain"`
MaxFilesAge int `long:"max-age" env:"MAX_AGE" default:"30" description:"maximum number of days to retain"`
MixErr bool `long:"mix-err" env:"MIX_ERR" description:"send error to std output log file"`
FilesLocation string `long:"loc" env:"LOG_FILES_LOC" default:"logs" description:"log files locations"`
FilePostfixExcludePattern string `long:"exclude-postfix" env:"FILE_POSTFIX_EXCLUDE_PATTERN" default:"" description:"exclude postfix from log file names with regexp pattern"` //nolint:lll

Excludes []string `short:"x" long:"exclude" env:"EXCLUDE" env-delim:"," description:"excluded container names"`
Includes []string `short:"i" long:"include" env:"INCLUDE" env-delim:"," description:"included container names"`
Expand Down Expand Up @@ -96,6 +97,17 @@ func do(ctx context.Context, opts *cliOpts) error {
}
}

if opts.FilePostfixExcludePattern != "" {
if opts.EnableFiles {
_, err := regexp.Compile(opts.FilePostfixExcludePattern)
if err != nil {
return errors.New("could not parse FilePostfixExcludePattern")
}
} else {
return errors.New("FilePostfixExcludePattern is only allowed when logging to files")
}
}

if opts.EnableSyslog && !syslog.IsSupported() {
return errors.New("syslog is not supported on this OS")
}
Expand All @@ -105,7 +117,8 @@ func do(ctx context.Context, opts *cliOpts) error {
return errors.Wrapf(err, "failed to make docker client %s", err)
}

events, err := discovery.NewEventNotif(client, opts.Excludes, opts.Includes, opts.IncludesPattern, opts.ExcludesPattern)
events, err := discovery.NewEventNotif(client, opts.Excludes, opts.Includes, opts.IncludesPattern, opts.ExcludesPattern,
opts.FilePostfixExcludePattern)
if err != nil {
return errors.Wrap(err, "failed to make event notifier")
}
Expand All @@ -127,7 +140,7 @@ func runEventLoop(ctx context.Context, opts *cliOpts, events *discovery.EventNot
return
}

logWriter, errWriter := makeLogWriters(opts, event.ContainerName, event.Group)
logWriter, errWriter := makeLogWriters(opts, event.ContainerName, event.FileName, event.Group)
ls := logger.LogStreamer{
DockerClient: client,
ContainerID: event.ContainerID,
Expand Down Expand Up @@ -183,7 +196,7 @@ func runEventLoop(ctx context.Context, opts *cliOpts, events *discovery.EventNot
// makeLogWriters creates io.Writer with rotated out and separate err files. Also adds writer for remote syslog
//
//nolint:funlen
func makeLogWriters(opts *cliOpts, containerName, group string) (logWriter, errWriter io.WriteCloser) {
func makeLogWriters(opts *cliOpts, containerName, fileName, group string) (logWriter, errWriter io.WriteCloser) {
log.Printf("[DEBUG] create log writer for %s", strings.TrimPrefix(group+"/"+containerName, "/"))
if !opts.EnableFiles && !opts.EnableSyslog {
log.Fatalf("[ERROR] either files or syslog has to be enabled")
Expand All @@ -201,7 +214,7 @@ func makeLogWriters(opts *cliOpts, containerName, group string) (logWriter, errW
log.Fatalf("[ERROR] can't make directory %s, %v", logDir, err)
}

logName := fmt.Sprintf("%s/%s.log", logDir, containerName)
logName := fmt.Sprintf("%s/%s.log", logDir, fileName)
logFileWriter := &lumberjack.Logger{
Filename: logName,
MaxSize: opts.MaxFileSize, // megabytes
Expand All @@ -215,7 +228,7 @@ func makeLogWriters(opts *cliOpts, containerName, group string) (logWriter, errW
errFname := logName

if !opts.MixErr { // if writers not mixed make error writer
errFname = fmt.Sprintf("%s/%s.err", logDir, containerName)
errFname = fmt.Sprintf("%s/%s.err", logDir, fileName)
errFileWriter = &lumberjack.Logger{
Filename: errFname,
MaxSize: opts.MaxFileSize, // megabytes
Expand Down
47 changes: 42 additions & 5 deletions app/main_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ func Test_makeLogWriters(t *testing.T) {
setupLog(true)

opts := cliOpts{FilesLocation: "/tmp/logger.test", EnableFiles: true, MaxFileSize: 1, MaxFilesCount: 10}
stdWr, errWr := makeLogWriters(&opts, "container1", "gr1")
stdWr, errWr := makeLogWriters(&opts, "container1", "container1", "gr1")
assert.NotEqual(t, stdWr, errWr, "different writers for out and err")

// write to out writer
Expand All @@ -63,12 +63,49 @@ func Test_makeLogWriters(t *testing.T) {
assert.NoError(t, errWr.Close())
}

func Test_makeLogWritersToTheSameFile(t *testing.T) {
defer os.RemoveAll("/tmp/logger.test") // nolint
setupLog(true)

// test with excluded postfix number after underscore
opts := cliOpts{FilesLocation: "/tmp/logger.test", EnableFiles: true, MaxFileSize: 1, MaxFilesCount: 10}
stdWr, errWr := makeLogWriters(&opts, "container1_1", "container1", "gr1")
assert.NotEqual(t, stdWr, errWr, "different writers for out and err")
stdWr2, errWr2 := makeLogWriters(&opts, "container1_2", "container1", "gr1")
assert.NotEqual(t, stdWr2, errWr2, "different writers for out and err")

// write to out writers
_, err := stdWr.Write([]byte("abc line 1 from container 1\n"))
assert.NoError(t, err)
_, err = stdWr2.Write([]byte("abc line 2 from container 2\n"))
assert.NoError(t, err)

// write to err writers
_, err = errWr.Write([]byte("err line 1 from container 1\n"))
assert.NoError(t, err)
_, err = errWr2.Write([]byte("err line 2 from container 2\n"))
assert.NoError(t, err)

r, err := os.ReadFile("/tmp/logger.test/gr1/container1.log")
assert.NoError(t, err)
assert.Equal(t, "abc line 1 from container 1\nabc line 2 from container 2\n", string(r))

r, err = os.ReadFile("/tmp/logger.test/gr1/container1.err")
assert.NoError(t, err)
assert.Equal(t, "err line 1 from container 1\nerr line 2 from container 2\n", string(r))

assert.NoError(t, stdWr.Close())
assert.NoError(t, stdWr2.Close())
assert.NoError(t, errWr.Close())
assert.NoError(t, errWr2.Close())
}

func Test_makeLogWritersMixed(t *testing.T) {
defer os.RemoveAll("/tmp/logger.test") // nolint
setupLog(false)

opts := cliOpts{FilesLocation: "/tmp/logger.test", EnableFiles: true, MaxFileSize: 1, MaxFilesCount: 10, MixErr: true}
stdWr, errWr := makeLogWriters(&opts, "container1", "gr1")
stdWr, errWr := makeLogWriters(&opts, "container1", "container1", "gr1")
assert.Equal(t, stdWr, errWr, "same writer for out and err in mixed mode")

// write to out writer
Expand All @@ -94,7 +131,7 @@ func Test_makeLogWritersMixed(t *testing.T) {
func Test_makeLogWritersWithJSON(t *testing.T) {
defer os.RemoveAll("/tmp/logger.test") // nolint
opts := cliOpts{FilesLocation: "/tmp/logger.test", EnableFiles: true, MaxFileSize: 1, MaxFilesCount: 10, ExtJSON: true}
stdWr, errWr := makeLogWriters(&opts, "container1", "gr1")
stdWr, errWr := makeLogWriters(&opts, "container1", "container1", "gr1")

// write to out writer
_, err := stdWr.Write([]byte("abc line 1"))
Expand All @@ -113,7 +150,7 @@ func Test_makeLogWritersWithJSON(t *testing.T) {

func Test_makeLogWritersSyslogFailed(t *testing.T) {
opts := cliOpts{EnableSyslog: true}
stdWr, errWr := makeLogWriters(&opts, "container1", "gr1")
stdWr, errWr := makeLogWriters(&opts, "container1", "container1", "gr1")
assert.Equal(t, stdWr, errWr, "same writer for out and err in syslog")
// write to out writer
_, err := stdWr.Write([]byte("abc line 1\n"))
Expand All @@ -130,7 +167,7 @@ func Test_makeLogWritersSyslogFailed(t *testing.T) {

func Test_makeLogWritersSyslogPassed(t *testing.T) {
opts := cliOpts{EnableSyslog: true, SyslogHost: "127.0.0.1:514", SyslogPrefix: "docker/"}
stdWr, errWr := makeLogWriters(&opts, "container1", "gr1")
stdWr, errWr := makeLogWriters(&opts, "container1", "container1", "gr1")
assert.Equal(t, stdWr, errWr, "same writer for out and err in syslog")

// write to out writer
Expand Down