Merge pull request #152 from Geta/feature/ignore-suggestions-regex

Feature/ignore suggestions regex

Author: jevgenijsp

README.md

@@ -159,6 +159,20 @@ By default, requests to files with the following extensions will be ignored by t
 
 If you want to specify this yourself, add `IgnoredResourceExtensions` to the configuration.
 
+### Specifying ignored URLs
+
+If certain URLs should be ignored, you can use the `IgnoreSuggestionsUrlRegexPattern` option:
+
+```
+services.AddNotFoundHandler(o =>
+{
+    o.IgnoreSuggestionsUrlRegexPattern = @"^(https?:\/\/[^\/]+)?\/(api|episerverapi|globalassets|siteassets)";
+});
+```
+
+When a URL matches the specified regex pattern, suggestions will be skipped.
+
+
 ## Restricting access to the Admin UI
 
 By default, only users of `Administrators` role can access Admin UI. But you can configure your authorization policy when registering the NotFound handler.

src/Geta.NotFoundHandler/Core/RequestHandler.cs

@@ -154,11 +154,11 @@ public virtual bool HandleRequest(Uri referer, Uri urlNotFound, out CustomRedire
             }
             else
             {
-                // log request to database - if logging is turned on.
-                if (_configuration.Logging == LoggerMode.On)
+                // Safe logging
+                var logUrl = _configuration.LogWithHostname ? urlNotFound.ToString() : urlNotFound.PathAndQuery;
+
+                if (_requestLogger.ShouldLogRequest(logUrl))
                 {
-                    // Safe logging
-                    var logUrl = _configuration.LogWithHostname ? urlNotFound.ToString() : urlNotFound.PathAndQuery;
                     _requestLogger.LogRequest(logUrl, referer?.ToString());
                 }
             }

src/Geta.NotFoundHandler/Core/Suggestions/IRequestLogger.cs

@@ -6,5 +6,6 @@ namespace Geta.NotFoundHandler.Core.Suggestions
     public interface IRequestLogger
     {
         void LogRequest(string oldUrl, string referer);
+        bool ShouldLogRequest(string oldUrl);
     }
-}
+}

src/Geta.NotFoundHandler/Core/Suggestions/RequestLogger.cs

@@ -4,6 +4,7 @@
 using System;
 using System.Collections.Concurrent;
 using System.Linq;
+using System.Text.RegularExpressions;
 using Geta.NotFoundHandler.Data;
 using Geta.NotFoundHandler.Infrastructure.Configuration;
 using Microsoft.Extensions.Logging;
@@ -27,7 +28,7 @@ public RequestLogger(
             _configuration = options.Value;
         }
 
-        public void LogRequest(string oldUrl, string referer)
+        public virtual void LogRequest(string oldUrl, string referer)
         {
             var bufferSize = _configuration.BufferSize;
             if (!LogQueue.IsEmpty && LogQueue.Count >= bufferSize)
@@ -80,6 +81,36 @@ private void LogRequests(ConcurrentQueue<LogEvent> logEvents)
             }
         }
 
+        public virtual bool ShouldLogRequest(string url)
+        {
+            // log request to database - if logging is turned on.
+            if (_configuration.Logging == LoggerMode.Off)
+            {
+                return false;
+            }
+
+            var ignorePattern = _configuration.IgnoreSuggestionsUrlRegexPattern;
+            if (string.IsNullOrWhiteSpace(ignorePattern))
+            {
+                return true;
+            }
+
+            try
+            {
+                return !Regex.IsMatch(url, ignorePattern, RegexOptions.None, TimeSpan.FromMilliseconds(50));
+            }
+            catch (RegexMatchTimeoutException ex)
+            {
+                _logger.LogWarning(ex, "Regex matching timed out for pattern: {Pattern} and URL: {Url}", ignorePattern, url);
+                return true;
+            }
+            catch (Exception ex)
+            {
+                _logger.LogWarning(ex, "Unexpected error while matching regex for URL: {Url} and pattern: {Pattern}", url, ignorePattern);
+                return true;
+            }
+        }
+
         private static ConcurrentQueue<LogEvent> LogQueue { get; } = new ConcurrentQueue<LogEvent>();
     }
 }

src/Geta.NotFoundHandler/Infrastructure/Configuration/NotFoundHandlerOptions.cs

@@ -26,6 +26,7 @@ public class NotFoundHandlerOptions
             {"jpg", "gif", "png", "css", "js", "ico", "swf", "woff"};
 
         public LoggerMode Logging { get; set; } = LoggerMode.On;
+        public string IgnoreSuggestionsUrlRegexPattern { get; set; }
         public bool LogWithHostname { get; set; } = false;
         public string ConnectionString { get; private set; }
         public string BootstrapJsUrl { get; set; } = "https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.bundle.min.js";

tests/Geta.NotFoundHandler.Tests/RequestHandlerTests.cs

@@ -4,6 +4,7 @@
 using Geta.NotFoundHandler.Core;
 using Geta.NotFoundHandler.Core.Redirects;
 using Geta.NotFoundHandler.Core.Suggestions;
+using Geta.NotFoundHandler.Data;
 using Geta.NotFoundHandler.Infrastructure.Configuration;
 using Geta.NotFoundHandler.Tests.Base;
 using Microsoft.AspNetCore.Http;
@@ -23,16 +24,23 @@ public class RequestHandlerTests
         private static readonly Uri DefaultOldUri = new Uri("http://example.com/old");
         private readonly NotFoundHandlerOptions _configuration;
 
+        private const string IgnoreSuggestionsUrlRegexPattern = @"^(https?:\/\/[^\/]+)?\/(api|nilleapi|episerverapi|globalassets|siteassets)(?!\/private)(\/.*)?$";
+
         public RequestHandlerTests()
         {
             _redirectHandler = A.Fake<IRedirectHandler>();
-            _requestLogger = A.Fake<IRequestLogger>();
             var options = A.Fake<IOptions<NotFoundHandlerOptions>>();
             _configuration = new NotFoundHandlerOptions();
             A.CallTo(() => options.Value).Returns(_configuration);
-            var logger = A.Fake<ILogger<RequestHandler>>();
+            var suggestionsRepository = A.Fake<ISuggestionRepository>();
+            var requestLoggerLogger = A.Fake<ILogger<RequestLogger>>();
+
+            _requestLogger = A.Fake<RequestLogger>(o => o.WithArgumentsForConstructor(new object[] {options, requestLoggerLogger, suggestionsRepository })
+                                                       .CallsBaseMethods());
+
+            var requestHandlerLogger = A.Fake<ILogger<RequestHandler>>();
             _sut = A.Fake<RequestHandler>(
-                o => o.WithArgumentsForConstructor(new object[] { _redirectHandler, _requestLogger, options, logger })
+                o => o.WithArgumentsForConstructor(new object[] { _redirectHandler, _requestLogger, options, requestHandlerLogger })
                     .CallsBaseMethods());
 
             _httpContext = new DefaultHttpContext();
@@ -255,6 +263,51 @@ private void AssertRequestLogged(Uri referrer, Uri urlNotFound)
                 .MustHaveHappened();
         }
 
+        [Theory]
+        [InlineData("https://example.com/api/something.json")]
+        [InlineData("/episerverapi/content")]
+        [InlineData("/globalassets/image.jpg")]
+        [InlineData("/siteassets/style.css")]
+        [InlineData("/api/resource")]
+        public void HandleRequest_ignore_suggestions_when_regex_matches(string url)
+        {
+            WhenLoggingIsOn();
+
+            _configuration.IgnoreSuggestionsUrlRegexPattern = IgnoreSuggestionsUrlRegexPattern;
+
+            var result = _requestLogger.ShouldLogRequest(url);
+
+            Assert.False(result);
+        }
+
+        [Theory]
+        [InlineData("https://example.com/home")]
+        [InlineData("/")]
+        [InlineData("/login")]
+        public void HandleRequest_do_not_ignore_suggestions_when_regex_do_not_matches(string url)
+        {
+            WhenLoggingIsOn();
+
+            _configuration.IgnoreSuggestionsUrlRegexPattern = IgnoreSuggestionsUrlRegexPattern;
+
+            var result = _requestLogger.ShouldLogRequest(url);
+
+            Assert.True(result);
+        }
+
+        [Theory]
+        [InlineData("/siteassets/private/file.json")]
+        public void HandleRequest_do_not_ignore_suggestions_when_regex_matches_using_lookahead(string url)
+        {
+            WhenLoggingIsOn();
+
+            _configuration.IgnoreSuggestionsUrlRegexPattern = IgnoreSuggestionsUrlRegexPattern;
+
+            var result = _requestLogger.ShouldLogRequest(url);
+
+            Assert.True(result);
+        }
+
         private void WhenLoggingIsOn()
         {
             _configuration.Logging = LoggerMode.On;