feat: log filtering

cmd/anubis/main.go

@@ -31,6 +31,7 @@ import (
 	"github.com/TecharoHQ/anubis/data"
 	"github.com/TecharoHQ/anubis/internal"
 	libanubis "github.com/TecharoHQ/anubis/lib"
+	"github.com/TecharoHQ/anubis/lib/logging"
 	botPolicy "github.com/TecharoHQ/anubis/lib/policy"
 	"github.com/TecharoHQ/anubis/lib/policy/config"
 	"github.com/TecharoHQ/anubis/lib/thoth"
@@ -250,7 +251,7 @@ func main() {
 		return
 	}
 
-	internal.InitSlog(*slogLevel)
+	slog.SetDefault(slog.New(logging.Init(*slogLevel)))
 	internal.SetHealth("anubis", healthv1.HealthCheckResponse_NOT_SERVING)
 
 	if *healthcheck {
@@ -447,7 +448,10 @@ func main() {
 	h = internal.XForwardedForUpdate(*xffStripPrivate, h)
 	h = internal.JA4H(h)
 
-	srv := http.Server{Handler: h, ErrorLog: internal.GetFilteredHTTPLogger()}
+	srv := http.Server{
+		Handler:  h,
+		ErrorLog: logging.StdlibLogger(s.GetLogger("http-server").Handler(), slog.LevelDebug),
+	}
 	listener, listenerUrl := setupListener(*bindNetwork, *bind)
 	slog.Info(
 		"listening",
@@ -507,7 +511,10 @@ func metricsServer(ctx context.Context, done func()) {
 		}
 	})
 
-	srv := http.Server{Handler: mux, ErrorLog: internal.GetFilteredHTTPLogger()}
+	srv := http.Server{
+		Handler:  mux,
+		ErrorLog: logging.StdlibLogger(slog.With("subsystem", "metrics-server").Handler(), slog.LevelDebug),
+	}
 	listener, metricsUrl := setupListener(*metricsBindNetwork, *metricsBind)
 	slog.Debug("listening for metrics", "url", metricsUrl)
 

cmd/containerbuild/main.go

@@ -10,7 +10,7 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/TecharoHQ/anubis/internal"
+	"github.com/TecharoHQ/anubis/lib/logging"
 	"github.com/facebookgo/flagenv"
 )
 
@@ -28,7 +28,7 @@ func main() {
 	flagenv.Parse()
 	flag.Parse()
 
-	internal.InitSlog(*slogLevel)
+	slog.SetDefault(slog.New(logging.Init(*slogLevel)))
 
 	koDockerRepo := strings.TrimSuffix(*dockerRepo, "/"+filepath.Base(*dockerRepo))
 

data/botPolicies.yaml

@@ -132,6 +132,28 @@ dnsbl: false
 
 #       <!-- ... -->
 
+# Logging settings for Anubis
+logging:
+  # CEL log filters. Note that this is a very powerful feature and it is very easy to get
+  # yourself into trouble with this. Avoid using log filters unless you are running into
+  # circumstances like https://github.com/TecharoHQ/anubis/issues/942. This has a nonzero
+  # impact on logging, which spirals out into a more than zero impact on Anubis'
+  # performance and memory usage.
+  filters:
+    # Every filter must have a name and an expression. You can use the same expression
+    # syntax as you can with bots or thresholds.
+    #
+    # If the expression returns `true`, then the log line is filtered _out_.
+    - name: "http-stdlib"
+      # Log lines where the message starts with "http:" are filtered out.
+      expression: msg.startsWith("http:")
+    - name: "context-canceled"
+      # Log lines relating to context cancellation are filtered out.
+      expression: msg.contains("context canceled")
+    - name: "http-pipelining"
+      # Log lines relating to HTTP/1.1 pipelining being improperly handled are filtered out.
+      expression: msg.contains("Unsolicited response received on idle HTTP channel")
+
 # Open Graph passthrough configuration, see here for more information:
 # https://anubis.techaro.lol/docs/admin/configuration/open-graph/
 openGraph:

docs/docs/CHANGELOG.md

@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 <!-- This changes the project to: -->
 
 - Fix lock convoy problem in decaymap ([#1103](https://github.com/TecharoHQ/anubis/issues/1103))
+- [Log filtering](./admin/configuration/expressions.mdx#log-filtering) rules have been added. This allows users to write custom log filtering logic.
 - Document missing environment variables in installation guide: `SLOG_LEVEL`, `COOKIE_PREFIX`, `FORCED_LANGUAGE`, and `TARGET_DISABLE_KEEPALIVE` ([#1086](https://github.com/TecharoHQ/anubis/pull/1086))
 - Add validation warning when persistent storage is used without setting signing keys
 - Fixed `robots2policy` to properly group consecutive user agents into `any:` instead of only processing the last one ([#925](https://github.com/TecharoHQ/anubis/pull/925))

docs/docs/admin/configuration/expressions.mdx

@@ -99,6 +99,10 @@ For this rule, if a request comes in matching [the signature of the `go get` com
 
 Anubis exposes the following variables to expressions:
 
+### Bot expressions
+
+Bot expressions are used for evaluating [bot rules](../policies.mdx#bot-policies).
+
 | Name            | Type                  | Explanation                                                                                                                                   | Example                                                      |
 | :-------------- | :-------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------- | :----------------------------------------------------------- |
 | `headers`       | `map[string, string]` | The [headers](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers) of the request being processed.                            | `{"User-Agent": "Mozilla/5.0 Gecko/20100101 Firefox/137.0"}` |
@@ -182,6 +186,23 @@ Something to keep in mind about system load average is that it is not aware of t
 
 Also keep in mind that this does not account for other kinds of latency like I/O latency. A system can have its web applications unresponsive due to high latency from a MySQL server but still have that web application server report a load near or at zero.
 
+### Log filtering
+
+Log filters are run on every time Anubis logs data. These are high throughput filters and should be written with care.
+
+| Name    | Type                  | Explanation                                                                                            | Example                                 |
+| :------ | :-------------------- | :----------------------------------------------------------------------------------------------------- | --------------------------------------- |
+| `time`  | Timestamp             | The time that the log line was emitted.                                                                | `2025-08-18T06:45:38-04:00`             |
+| `msg`   | `string`              | The text-based message for the given log line.                                                         | `"invalid response"`                    |
+| `level` | `string`              | The [log level](https://pkg.go.dev/log/slog#Level) for the log message.                                | `"INFO"`                                |
+| `attrs` | `map[string, string]` | The key -> value attributes for the given log line. Note that this is an expensive variable to access. | `{"err": "internal: the sun exploded"}` |
+
+:::note
+
+When you define a log filter, anything matching that filter is _removed_. Any remaining logs are sent through to the system journal or standard error.
+
+:::
+
 ## Functions exposed to Anubis expressions
 
 Anubis expressions can be augmented with the following functions:

docs/docs/admin/policies.mdx

@@ -123,6 +123,10 @@ remote_addresses:
 
 Anubis has support for showing imprint / impressum information. This is defined in the `impressum` block of your configuration. See [Imprint / Impressum configuration](./configuration/impressum.mdx) for more information.
 
+## Logging
+
+Anubis has support for configuring log filtering using expressions. See the [log filters](./configuration/expressions.mdx#log-filters) of the [expression](./configuration/expressions.mdx) documentation for more information.
+
 ## Storage backends
 
 Anubis needs to store temporary data in order to determine if a user is legitimate or not. Administrators should choose a storage backend based on their infrastructure needs. Each backend has its own advantages and disadvantages.

internal/log.go

@@ -1,31 +1,10 @@
 package internal
 
 import (
-	"fmt"
-	"log"
 	"log/slog"
 	"net/http"
-	"os"
-	"strings"
 )
 
-func InitSlog(level string) {
-	var programLevel slog.Level
-	if err := (&programLevel).UnmarshalText([]byte(level)); err != nil {
-		fmt.Fprintf(os.Stderr, "invalid log level %s: %v, using info\n", level, err)
-		programLevel = slog.LevelInfo
-	}
-
-	leveler := &slog.LevelVar{}
-	leveler.Set(programLevel)
-
-	h := slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{
-		AddSource: true,
-		Level:     leveler,
-	})
-	slog.SetDefault(slog.New(h))
-}
-
 func GetRequestLogger(base *slog.Logger, r *http.Request) *slog.Logger {
 	host := r.Host
 	if host == "" {
@@ -44,27 +23,3 @@ func GetRequestLogger(base *slog.Logger, r *http.Request) *slog.Logger {
 		"x-real-ip", r.Header.Get("X-Real-Ip"),
 	)
 }
-
-// ErrorLogFilter is used to suppress "context canceled" logs from the http server when a request is canceled (e.g., when a client disconnects).
-type ErrorLogFilter struct {
-	Unwrap *log.Logger
-}
-
-func (elf *ErrorLogFilter) Write(p []byte) (n int, err error) {
-	logMessage := string(p)
-	if strings.Contains(logMessage, "context canceled") {
-		return len(p), nil // Suppress the log by doing nothing
-	}
-	if strings.Contains(logMessage, "Unsolicited response received on idle HTTP channel") {
-		return len(p), nil
-	}
-	if elf.Unwrap != nil {
-		return elf.Unwrap.Writer().Write(p)
-	}
-	return len(p), nil
-}
-
-func GetFilteredHTTPLogger() *log.Logger {
-	stdErrLogger := log.New(os.Stderr, "", log.LstdFlags) // essentially what the default logger is.
-	return log.New(&ErrorLogFilter{Unwrap: stdErrLogger}, "", 0)
-}

lib/anubis.go

@@ -78,6 +78,10 @@ type Server struct {
 	logger      *slog.Logger
 }
 
+func (s *Server) GetLogger(subsystem string) *slog.Logger {
+	return s.logger.With("subsystem", subsystem)
+}
+
 func (s *Server) getTokenKeyfunc() jwt.Keyfunc {
 	// return ED25519 key if HS512 is not set
 	if len(s.hs512Secret) == 0 {

lib/config.go

@@ -120,6 +120,14 @@ func New(opts Options) (*Server, error) {
 		logger:      opts.Logger,
 	}
 
+	if opts.Policy.Logging != nil {
+		var err error
+		result.logger, err = opts.Policy.ApplyLogFilters(opts.Logger)
+		if err != nil {
+			return nil, fmt.Errorf("can't create log filters: %w", err)
+		}
+	}
+
 	mux := http.NewServeMux()
 	xess.Mount(mux)
 

lib/logging/filter.go

@@ -0,0 +1,67 @@
+package logging
+
+import (
+	"context"
+	"log/slog"
+)
+
+// Filterer is the shape of any type that can perform log filtering. This takes
+// the context of the log filtering call and the log record to be filtered.
+type Filterer interface {
+	Filter(ctx context.Context, r slog.Record) bool
+}
+
+// FilterFunc lets you make inline log filters with plain functions.
+type FilterFunc func(ctx context.Context, r *slog.Record) bool
+
+// Filter implements Filterer for FilterFunc.
+func (ff FilterFunc) Filter(ctx context.Context, r *slog.Record) bool {
+	return ff(ctx, r)
+}
+
+// FilterHandler wraps a slog Handler with one or more filters, enabling administrators
+// to customize the logging subsystem of Anubis.
+type FilterHandler struct {
+	next    slog.Handler
+	filters []Filterer
+}
+
+// NewFilterHandler creates a new filtering handler with the given base handler and filters.
+func NewFilterHandler(handler slog.Handler, filters ...Filterer) *FilterHandler {
+	return &FilterHandler{
+		next:    handler,
+		filters: filters,
+	}
+}
+
+// Enabled passes through to the upstream slog Handler.
+func (h *FilterHandler) Enabled(ctx context.Context, level slog.Level) bool {
+	return h.next.Enabled(ctx, level)
+}
+
+// Handle implements slog.Handler and applies all filters before delegating to the base handler.
+func (h *FilterHandler) Handle(ctx context.Context, r slog.Record) error {
+	// Apply all filters - if any filter returns false, skip the log
+	for _, filter := range h.filters {
+		if !filter.Filter(ctx, r) {
+			return nil // Skip this log record
+		}
+	}
+	return h.next.Handle(ctx, r)
+}
+
+// WithAttrs implements slog.Handler.
+func (h *FilterHandler) WithAttrs(attrs []slog.Attr) slog.Handler {
+	return &FilterHandler{
+		next:    h.next.WithAttrs(attrs),
+		filters: h.filters,
+	}
+}
+
+// WithGroup implements slog.Handler.
+func (h *FilterHandler) WithGroup(name string) slog.Handler {
+	return &FilterHandler{
+		next:    h.next.WithGroup(name),
+		filters: h.filters,
+	}
+}