Add JSON schema validation for rules and tests (#626)

* Add JSON schema validation for rules and tests

Introduces JSON schema validation for Application Inspector rules using JsonSchema.Net. Adds RuleSchemaProvider and rule-schema.json, updates RulesVerifier and RuleStatus to support schema validation, and extends RulesVerifierOptions for configuration. Includes unit tests for schema validation in SchemaValidationTests.

* Update AppInspector.RulesEngine/Schema/RuleSchemaProvider.cs

Co-authored-by: Copilot <[email protected]>

* Update rule schema and improve schema validation tests

Changed 'must-match' and 'must-not-match' in rule-schema.json to arrays of strings with updated descriptions. Enhanced SchemaValidationTests to validate all embedded default rules using RuleSetUtils and provide detailed error reporting for failed schema validations.

* Cache schema validation results in rules

Adds a SchemaValidationResult property to Rule and updates TypedRuleSet to validate and store schema results when loading rules from JSON. RulesVerifier now uses the cached schema validation result if available, avoiding redundant validations. Also updates constructors to pass the schema provider and simplifies JSON serialization in RuleSchemaProvider.

* Remove line number and position from schema errors

Eliminated the inclusion of line number and position details from schema validation error and warning messages in RulesVerifier. Also removed the corresponding properties from the RuleSchemaProvider error class, simplifying error reporting.

* Remove debug code from schema validation test

Eliminated commented-out and unused debug code related to JSON serialization and error output in SchemaValidationTests. This cleanup improves test readability and removes unnecessary code.

* Add negative schema validation tests for rules

Introduces multiple unit tests to verify that invalid rule definitions fail schema validation as expected. Tests cover invalid pattern types, confidence levels, severities, scopes, empty arrays, missing required fields, and schema validation error handling at both error and warning levels.

* Update AppInspector.Tests/RuleProcessor/SchemaValidationTests.cs

Co-authored-by: Copilot <[email protected]>

* Update rule schema handling and add versioned schema

Moved rule-schema.json to the project root and updated its $id. Added rule-schema-v1.json as a new versioned schema file. Updated project and code references to use the new rule-schema-v1.json as the embedded resource.

* Update schema validation tests and remove rule-schema.json

Replaces deprecated Assert.Empty with Assert.Fail in SchemaValidationTests and simplifies type references for schema validation result and error classes. Also removes the rule-schema.json file, possibly indicating a migration to a different schema management approach.

* Refactor error collection to use stack instead of recursion

Replaces recursive calls in CollectErrorsFromResult with an explicit stack to avoid potential stack overflow on deeply nested validation results. This change improves robustness when processing large or deeply nested schemas.

---------

Co-authored-by: Copilot <[email protected]>

Author: gfs

AppInspector.RulesEngine/AppInspector.RulesEngine.csproj

@@ -31,6 +31,7 @@
 
     <ItemGroup>
         <PackageReference Include="JsonCons.JsonPath" Version="1.1.0" />
+        <PackageReference Include="JsonSchema.Net" Version="7.2.3" />
         <PackageReference Include="Microsoft.CST.OAT" Version="1.2.87" />
         <PackageReference Include="Microsoft.CST.RecursiveExtractor" Version="1.2.45" />
         <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="9.0.8" />
@@ -45,6 +46,7 @@
     <ItemGroup>
         <EmbeddedResource Include="Resources\comments.json" />
         <EmbeddedResource Include="Resources\languages.json" />
+        <EmbeddedResource Include="..\rule-schema-v1.json" />
     </ItemGroup>
 
     <ItemGroup>

AppInspector.RulesEngine/Rule.cs

@@ -41,6 +41,12 @@ public class Rule
         [JsonIgnore]
         public bool Disabled { get; set; }
 
+        /// <summary>
+        ///     Schema validation result from when the rule was loaded from JSON
+        /// </summary>
+        [JsonIgnore]
+        public Schema.SchemaValidationResult? SchemaValidationResult { get; set; }
+
         /// <summary>
         /// Tags that are required to be present in the total result set - even from other rules matched against other files - for this match to be valid
         /// Does not work with `TagsOnly` option.

AppInspector.RulesEngine/RuleStatus.cs

@@ -1,5 +1,6 @@
 using System.Collections.Generic;
 using System.Linq;
+using Microsoft.ApplicationInspector.RulesEngine.Schema;
 using Microsoft.CST.OAT;
 
 namespace Microsoft.ApplicationInspector.RulesEngine;
@@ -8,10 +9,20 @@ public class RuleStatus
 {
     public string? RulesId { get; set; }
     public string? RulesName { get; set; }
-    public bool Verified => !Errors.Any() && !OatIssues.Any();
+    public bool Verified => !Errors.Any() && !OatIssues.Any() && !SchemaValidationErrors.Any();
     public IEnumerable<string> Errors { get; set; } = Enumerable.Empty<string>();
     public IEnumerable<Violation> OatIssues { get; set; } = Enumerable.Empty<Violation>();
     public bool HasPositiveSelfTests { get; set; }
     public bool HasNegativeSelfTests { get; set; }
-    internal Rule Rule { get; set; }
+    internal Rule? Rule { get; set; }
+    
+    /// <summary>
+    /// Schema validation errors for this rule
+    /// </summary>
+    public IEnumerable<SchemaValidationError> SchemaValidationErrors { get; set; } = Enumerable.Empty<SchemaValidationError>();
+    
+    /// <summary>
+    /// Whether the rule passed schema validation
+    /// </summary>
+    public bool PassedSchemaValidation { get; set; } = true;
 }

AppInspector.RulesEngine/RulesVerifier.cs

@@ -10,6 +10,7 @@
 using JsonCons.JsonPath;
 using Microsoft.ApplicationInspector.Common;
 using Microsoft.ApplicationInspector.RulesEngine.OatExtensions;
+using Microsoft.ApplicationInspector.RulesEngine.Schema;
 using Microsoft.CST.OAT;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Logging.Abstractions;
@@ -24,12 +25,18 @@ public class RulesVerifier
     private readonly Analyzer _analyzer;
     private readonly ILogger _logger;
     private readonly RulesVerifierOptions _options;
+    private readonly RuleSchemaProvider? _schemaProvider;
 
     public RulesVerifier(RulesVerifierOptions options)
     {
         _options = options;
         _logger = _options.LoggerFactory?.CreateLogger<RulesVerifier>() ?? NullLogger<RulesVerifier>.Instance;
         _analyzer = _options.Analyzer ?? new ApplicationInspectorAnalyzer(_options.LoggerFactory);
+        
+        if (_options.EnableSchemaValidation)
+        {
+            _schemaProvider = _options.SchemaProvider ?? new RuleSchemaProvider(_options.CustomSchemaPath);
+        }
     }
 
     private ILoggerFactory? _loggerFactory => _options.LoggerFactory;
@@ -42,7 +49,7 @@ public RulesVerifier(RulesVerifierOptions options)
     /// <exception cref="OpException"></exception>
     public RulesVerifierResult Verify(string rulesPath)
     {
-        RuleSet CompiledRuleset = new(_loggerFactory);
+        RuleSet CompiledRuleset = new(_loggerFactory, _schemaProvider);
 
         if (!string.IsNullOrEmpty(rulesPath))
         {
@@ -139,9 +146,53 @@ public IList<RuleStatus> CheckIntegrity(AbstractRuleSet ruleSet)
     public RuleStatus CheckIntegrity(ConvertedOatRule convertedOatRule)
     {
         List<string> errors = new();
+        List<SchemaValidationError> schemaErrors = new();
+        bool passedSchemaValidation = true;
 
         // App Inspector checks
         var rule = convertedOatRule.AppInspectorRule;
+
+        // Schema validation step (use stored result from rule loading if available)
+        if (_options.EnableSchemaValidation && _schemaProvider != null)
+        {
+            SchemaValidationResult validationResult;
+            
+            // Use the stored schema validation result from rule loading if available
+            if (rule.SchemaValidationResult != null)
+            {
+                validationResult = rule.SchemaValidationResult;
+            }
+            else
+            {
+                // Fallback to individual rule validation (inefficient)
+                _logger.LogDebug("No stored schema validation result for rule {RuleId}, performing re-validation", rule.Id);
+                validationResult = _schemaProvider.ValidateRule(rule);
+            }
+            
+            schemaErrors.AddRange(validationResult.Errors);
+            passedSchemaValidation = validationResult.IsValid;
+
+            if (!validationResult.IsValid)
+            {
+                if (_options.SchemaValidationLevel == SchemaValidationLevel.Error)
+                {
+                    foreach (var error in validationResult.Errors)
+                    {
+                        var errorMessage = $"Schema validation error at {error.Path}: {error.Message}";
+                        errors.Add(errorMessage);
+                        _logger.LogError("Schema validation error for rule {RuleId}: {Error}", rule.Id ?? "Unknown", errorMessage);
+                    }
+                }
+                else if (_options.SchemaValidationLevel == SchemaValidationLevel.Warning)
+                {
+                    foreach (var error in validationResult.Errors)
+                    {
+                        var errorMessage = $"Schema validation warning at {error.Path}: {error.Message}";
+                        _logger.LogWarning("Schema validation warning for rule {RuleId}: {Error}", rule.Id ?? "Unknown", errorMessage);
+                    }
+                }
+            }
+        }
         // Check for null Id
         if (string.IsNullOrEmpty(rule.Id))
         {
@@ -383,7 +434,9 @@ public RuleStatus CheckIntegrity(ConvertedOatRule convertedOatRule)
             Errors = errors,
             OatIssues = _analyzer.EnumerateRuleIssues(convertedOatRule),
             HasPositiveSelfTests = rule.MustMatch?.Count > 0,
-            HasNegativeSelfTests = rule.MustNotMatch?.Count > 0
+            HasNegativeSelfTests = rule.MustNotMatch?.Count > 0,
+            SchemaValidationErrors = schemaErrors,
+            PassedSchemaValidation = passedSchemaValidation
         };
     }
 }

AppInspector.RulesEngine/RulesVerifierOptions.cs

@@ -1,4 +1,5 @@
-using Microsoft.CST.OAT;
+using Microsoft.ApplicationInspector.RulesEngine.Schema;
+using Microsoft.CST.OAT;
 using Microsoft.Extensions.Logging;
 
 namespace Microsoft.ApplicationInspector.RulesEngine;
@@ -38,4 +39,24 @@ public class RulesVerifierOptions
     public bool RequireMustNotMatch { get; set; }
 
     public bool EnableNonBacktrackingRegex { get; set; }
+
+    /// <summary>
+    ///     Enable JSON schema validation for rules
+    /// </summary>
+    public bool EnableSchemaValidation { get; set; } = false;
+
+    /// <summary>
+    ///     Custom schema provider for validation
+    /// </summary>
+    public RuleSchemaProvider? SchemaProvider { get; set; }
+
+    /// <summary>
+    ///     How to handle schema validation failures
+    /// </summary>
+    public SchemaValidationLevel SchemaValidationLevel { get; set; } = SchemaValidationLevel.Warning;
+
+    /// <summary>
+    ///     Path to custom JSON schema file (optional)
+    /// </summary>
+    public string? CustomSchemaPath { get; set; }
 }

AppInspector.RulesEngine/Ruleset.cs

@@ -1,6 +1,7 @@
 // Copyright (C) Microsoft. All rights reserved. Licensed under the MIT License.
 
 
+using Microsoft.ApplicationInspector.RulesEngine.Schema;
 using Microsoft.Extensions.Logging;
 
 namespace Microsoft.ApplicationInspector.RulesEngine;
@@ -11,11 +12,12 @@ namespace Microsoft.ApplicationInspector.RulesEngine;
 public class RuleSet : TypedRuleSet<Rule>
 {
     /// <summary>
-    ///     Create a ruleset using the given (optional) logger.
+    ///     Create a ruleset using the given (optional) logger and schema provider.
     /// </summary>
     /// <param name="loggerFactory"></param>
-    public RuleSet(ILoggerFactory? loggerFactory = null) 
-        : base(loggerFactory)
+    /// <param name="schemaProvider"></param>
+    public RuleSet(ILoggerFactory? loggerFactory = null, RuleSchemaProvider? schemaProvider = null) 
+        : base(loggerFactory, schemaProvider)
     {    
     }
 }