Merge pull request #334 from Azure/johnathon/sarifSeverity

Include rule severities when outputting results to SARIF
Azure · Apr 27, 2023 · 602e7ff · 602e7ff
2 parents 6b317dd + 1dba1d4
commit 602e7ff
Show file tree

Hide file tree

Showing 7 changed files with 196 additions and 105 deletions.
diff --git a/src/Analyzer.Reports.UnitTests/MockFileStream.cs b/src/Analyzer.Reports.UnitTests/MockFileStream.cs
diff --git a/src/Analyzer.Reports.UnitTests/SarifReportWriterE2ETests.cs b/src/Analyzer.Reports.UnitTests/SarifReportWriterE2ETests.cs
@@ -90,7 +90,7 @@ public void AnalyzeTemplateTests(string template, string referencedTemplate = nu
             foreach (Result result in run.Results)
             {
                 expectedLinesForRun.ContainsKey(result.RuleId).Should().BeTrue("Unexpected result found in SARIF");
-                result.Level.Should().Be(FailureLevel.Error);
+                result.Level.Should().Be(Utilities.GetLevelFromSeverity(results.First(r => result.RuleId == r.RuleId).Severity));
 
                 // Determine which template file was evaluated for this SARIF result (all locations will be in same file, so taking first)
                 // depending on if eval file matches the original target file, verify if analysis target is present or not
@@ -167,6 +167,7 @@ public void AnalyzeDirectoryTests(string firstTemplate, string secondTemplate, s
 
             // act
             var memStream = new MemoryStream();
+            List<Types.IEvaluation> analyzerResults = null;
             try
             {
                 // secondTemplateDirectory is always equal to or under targetDirectory
@@ -193,12 +194,14 @@ public void AnalyzeDirectoryTests(string firstTemplate, string secondTemplate, s
                     parameters: null,
                     templateFilePath: firstTemplateFileInfo.FullName);
                 writer.WriteResults(results, (FileInfoBase)firstTemplateFileInfo);
+                analyzerResults = results.ToList();
 
                 results = analyzer.AnalyzeTemplate(
                     template: secondTemplateString,
                     parameters: null,
                     templateFilePath: secondTemplateFileInfo.FullName);
                 writer.WriteResults(results, (FileInfoBase)secondTemplateFileInfo);
+                analyzerResults.AddRange(results.ToList());
             }
             finally
             {
@@ -352,7 +355,7 @@ public void AnalyzeDirectoryTests(string firstTemplate, string secondTemplate, s
                     expectedLinesForRun.Remove(result.RuleId);
                 }
 
-                result.Level.Should().Be(FailureLevel.Error);
+                result.Level.Should().Be(Utilities.GetLevelFromSeverity(analyzerResults.First(r => result.RuleId == r.RuleId).Severity));
             }
 
             // Verify all lines and results were reported

diff --git a/src/Analyzer.Reports.UnitTests/SarifReportWriterTests.cs b/src/Analyzer.Reports.UnitTests/SarifReportWriterTests.cs
@@ -114,7 +114,7 @@ private void AssertSarifLog(SarifLog sarifLog, IEnumerable<Types.IEvaluation> ev
                 foreach (var evaluation in evaluations)
                 {
                     var rule = rules.SingleOrDefault(r => r.Id.Equals(evaluation.RuleId));
-                    if (evaluation.Passed)
+                    if (evaluation.Passed && !failedEvaluations.Any(e => e.RuleId == evaluation.RuleId))
                     {
                         rule.Should().BeNull();
                     }
@@ -127,6 +127,11 @@ private void AssertSarifLog(SarifLog sarifLog, IEnumerable<Types.IEvaluation> ev
                         rule.FullDescription.Text.Should().BeEquivalentTo(SarifReportWriter.AppendPeriod(evaluation.RuleFullDescription));
                         rule.Help.Text.Should().BeEquivalentTo(SarifReportWriter.AppendPeriod(evaluation.Recommendation));
                         rule.HelpUri.OriginalString.Should().BeEquivalentTo(evaluation.HelpUri);
+
+                        // rule.DefaultConfiguration is not tested here. It appears to be primarily used internally in the SARIF SDK to determine levels for individual results.
+                        // For example, if DefaultConfiguration.Level is explicitly set to FailureLevel.Warning (and nothing else is set in DefaultConfiguration) for a rule,
+                        // then the DefaultConfiguration for that rule will be null in the resulting SARIF output file.
+                        // It's therefore not worth testing here, as the tests would have to account for the internal logic of the SARIF library itself (which may change at any time).
                     }
                 }
 
@@ -145,7 +150,8 @@ private void AssertSarifLog(SarifLog sarifLog, IEnumerable<Types.IEvaluation> ev
                     var result = run.Results[outputResults.Count];
                     result.RuleId.Should().BeEquivalentTo(evaluation.RuleId);
                     result.Message.Id.Should().BeEquivalentTo("default");
-                    result.Level.Should().Be(FailureLevel.Error);
+                    result.Kind.Should().Be(ResultKind.Fail);
+                    result.Level.Should().Be(Utilities.GetLevelFromSeverity(evaluation.Severity));
 
                     if (evalDistinctResults.First().SourceLocation.FilePath != templateFilePath.FullName)
                     {