Add redaction workflow logic (#581)

* Add redaction workflow logic

* Fix test

* PR feedback pt 1

* PR feedback pt 2

* PR feedback pt 3

* PR feedback pt 4

* PR feedback pt 5

src/Microsoft.Sbom.Api/FormatValidator/IValidatedSBOM.cs

@@ -0,0 +1,15 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Microsoft.Sbom.Api.FormatValidator;
+
+using System;
+using System.Threading.Tasks;
+using Microsoft.Sbom.Parsers.Spdx22SbomParser.Entities;
+
+public interface IValidatedSBOM: IDisposable
+{
+    public Task<FormatValidationResults> GetValidationResults();
+
+    public Task<FormatEnforcedSPDX2> GetRawSPDXDocument();
+}

src/Microsoft.Sbom.Api/FormatValidator/ValidatedSBOM.cs

@@ -13,7 +13,7 @@ namespace Microsoft.Sbom.Api.FormatValidator;
 using Microsoft.Sbom.Parsers.Spdx22SbomParser.Entities;
 using Microsoft.Sbom.Utils;
 
-public class ValidatedSBOM
+public class ValidatedSBOM: IValidatedSBOM
 {
     private readonly Stream sbomStream;
     private readonly int requiredSpdxMajorVersion = 2;
@@ -50,6 +50,13 @@ public async Task<FormatEnforcedSPDX2> GetRawSPDXDocument()
         return sbom;
     }
 
+    /// <inheritdoc/>
+    public void Dispose()
+    {
+        this.sbomStream?.Dispose();
+        GC.SuppressFinalize(this);
+    }
+
     private async Task Initialize()
     {
         if (isInitialized)

src/Microsoft.Sbom.Api/FormatValidator/ValidatedSBOMFactory.cs

@@ -0,0 +1,16 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Microsoft.Sbom.Api.FormatValidator;
+
+using System.IO;
+
+public class ValidatedSBOMFactory
+{
+    public virtual IValidatedSBOM CreateValidatedSBOM(string sbomFilePath)
+    {
+        var sbomStream = new StreamReader(sbomFilePath);
+        var validatedSbom = new ValidatedSBOM(sbomStream.BaseStream);
+        return validatedSbom;
+    }
+}

src/Microsoft.Sbom.Api/Workflows/Helpers/ISbomRedactor.cs

@@ -0,0 +1,16 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System.Threading.Tasks;
+using Microsoft.Sbom.Api.FormatValidator;
+using Microsoft.Sbom.Parsers.Spdx22SbomParser.Entities;
+
+namespace Microsoft.Sbom.Api.Workflows.Helpers;
+
+/// <summary>
+/// SBOM redactor that removes file information from SBOMs
+/// </summary>
+public interface ISbomRedactor
+{
+    public Task<FormatEnforcedSPDX2> RedactSBOMAsync(IValidatedSBOM sbom);
+}

src/Microsoft.Sbom.Api/Workflows/Helpers/SbomRedactor.cs

@@ -0,0 +1,101 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.Sbom.Api.FormatValidator;
+using Microsoft.Sbom.Common.Utils;
+using Microsoft.Sbom.Parsers.Spdx22SbomParser.Entities;
+using Serilog;
+
+namespace Microsoft.Sbom.Api.Workflows.Helpers;
+
+/// <summary>
+/// SBOM redactor that removes file information from SBOMs
+/// </summary>
+public class SbomRedactor: ISbomRedactor
+{
+    private const string SpdxFileRelationshipPrefix = "SPDXRef-File-";
+
+    private readonly ILogger log;
+
+    public SbomRedactor(
+        ILogger log)
+    {
+        this.log = log ?? throw new ArgumentNullException(nameof(log));
+    }
+
+    public virtual async Task<FormatEnforcedSPDX2> RedactSBOMAsync(IValidatedSBOM sbom)
+    {
+        var spdx = await sbom.GetRawSPDXDocument();
+
+        if (spdx.Files != null)
+        {
+            this.log.Debug("Removing files section from SBOM.");
+            spdx.Files = null;
+        }
+
+        RemovePackageFileRefs(spdx);
+        RemoveRelationshipsWithFileRefs(spdx);
+        UpdateDocumentNamespace(spdx);
+
+        return spdx;
+    }
+
+    private void RemovePackageFileRefs(FormatEnforcedSPDX2 spdx)
+    {
+        if (spdx.Packages != null)
+        {
+            foreach (var package in spdx.Packages)
+            {
+                if (package.HasFiles != null)
+                {
+                    this.log.Debug($"Removing has files property from package {package.Name}.");
+                    package.HasFiles = null;
+                }
+
+                if (package.SourceInfo != null)
+                {
+                    this.log.Debug($"Removing has sourceInfo property from package {package.Name}.");
+                    package.SourceInfo = null;
+                }
+            }
+        }
+    }
+
+    private void RemoveRelationshipsWithFileRefs(FormatEnforcedSPDX2 spdx)
+    {
+        if (spdx.Relationships != null)
+        {
+            var relationshipsToRemove = new List<SPDXRelationship>();
+            foreach (var relationship in spdx.Relationships)
+            {
+                if (relationship.SourceElementId.Contains(SpdxFileRelationshipPrefix) || relationship.TargetElementId.Contains(SpdxFileRelationshipPrefix))
+                {
+                    relationshipsToRemove.Add(relationship);
+                }
+            }
+
+            if (relationshipsToRemove.Any())
+            {
+                this.log.Debug($"Removing {relationshipsToRemove.Count()} relationships with file references from SBOM.");
+                spdx.Relationships = spdx.Relationships.Except(relationshipsToRemove);
+            }
+        }
+    }
+
+    private void UpdateDocumentNamespace(FormatEnforcedSPDX2 spdx)
+    {
+        if (!string.IsNullOrWhiteSpace(spdx.DocumentNamespace) && spdx.CreationInfo.Creators.Any(c => c.StartsWith("Tool: Microsoft.SBOMTool", StringComparison.OrdinalIgnoreCase)))
+        {
+            var existingNamespaceComponents = spdx.DocumentNamespace.Split('/');
+            var uniqueComponent = IdentifierUtils.GetShortGuid(Guid.NewGuid());
+            existingNamespaceComponents[^1] = uniqueComponent;
+            spdx.DocumentNamespace = string.Join("/", existingNamespaceComponents);
+
+            this.log.Debug($"Updated document namespace to {spdx.DocumentNamespace}.");
+        }
+    }
+}

src/Microsoft.Sbom.Api/Workflows/SBOMRedactionWorkflow.cs

@@ -2,7 +2,14 @@
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
 using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text.Json;
 using System.Threading.Tasks;
+using Microsoft.Sbom.Api.FormatValidator;
+using Microsoft.Sbom.Api.Workflows.Helpers;
+using Microsoft.Sbom.Common;
 using Microsoft.Sbom.Common.Config;
 using Serilog;
 
@@ -17,17 +24,121 @@ public class SbomRedactionWorkflow : IWorkflow<SbomRedactionWorkflow>
 
     private readonly IConfiguration configuration;
 
+    private readonly IFileSystemUtils fileSystemUtils;
+
+    private readonly ValidatedSBOMFactory validatedSBOMFactory;
+
+    private readonly ISbomRedactor sbomRedactor;
+
     public SbomRedactionWorkflow(
         ILogger log,
-        IConfiguration configuration)
+        IConfiguration configuration,
+        IFileSystemUtils fileSystemUtils,
+        ValidatedSBOMFactory validatedSBOMFactory,
+        ISbomRedactor sbomRedactor)
     {
         this.log = log ?? throw new ArgumentNullException(nameof(log));
         this.configuration = configuration ?? throw new ArgumentNullException(nameof(configuration));
+        this.fileSystemUtils = fileSystemUtils ?? throw new ArgumentNullException(nameof(fileSystemUtils));
+        this.validatedSBOMFactory = validatedSBOMFactory ?? throw new ArgumentNullException(nameof(validatedSBOMFactory));
+        this.sbomRedactor = sbomRedactor ?? throw new ArgumentNullException(nameof(sbomRedactor));
     }
 
     public virtual async Task<bool> RunAsync()
     {
-        log.Information($"Running redaction for SBOM path {configuration.SbomPath?.Value} and SBOM dir {configuration.SbomDir?.Value}. Output dir: {configuration.OutputPath?.Value}");
-        return await Task.FromResult(true);
+        ValidateDirStrucutre();
+        var sbomPaths = GetInputSbomPaths();
+        foreach (var sbomPath in sbomPaths)
+        {
+            IValidatedSBOM validatedSbom = null;
+            try
+            {
+                log.Information($"Validating SBOM {sbomPath}");
+                validatedSbom = validatedSBOMFactory.CreateValidatedSBOM(sbomPath);
+                var validationDetails = await validatedSbom.GetValidationResults();
+                if (validationDetails.Status != FormatValidationStatus.Valid)
+                {
+                    throw new InvalidDataException($"Failed to validate {sbomPath}:\n{string.Join('\n', validationDetails.Errors)}");
+                }
+                else
+                {
+                    log.Information($"Redacting SBOM {sbomPath}");
+                    var outputPath = GetOutputPath(sbomPath);
+                    var redactedSpdx = await this.sbomRedactor.RedactSBOMAsync(validatedSbom);
+                    using (var outStream = fileSystemUtils.OpenWrite(outputPath))
+                    {
+                        await JsonSerializer.SerializeAsync(outStream, redactedSpdx);
+                    }
+
+                    log.Information($"Redacted SBOM {sbomPath} saved to {outputPath}");
+                }
+            }
+            finally
+            {
+                validatedSbom?.Dispose();
+            }
+        }
+
+        return true;
+    }
+
+    private string GetOutputPath(string sbomPath)
+    {
+        return fileSystemUtils.JoinPaths(configuration.OutputPath.Value, fileSystemUtils.GetFileName(sbomPath));
+    }
+
+    private IEnumerable<string> GetInputSbomPaths()
+    {
+        if (configuration.SbomPath?.Value != null)
+        {
+            return new List<string>() { configuration.SbomPath.Value };
+        }
+        else if (configuration.SbomDir?.Value != null)
+        {
+            return fileSystemUtils.GetFilesInDirectory(configuration.SbomDir.Value);
+        }
+        else
+        {
+            throw new Exception("No valid input SBOMs to redact provided.");
+        }
+    }
+
+    private string ValidateDirStrucutre()
+    {
+        string inputDir;
+        if (configuration.SbomDir?.Value != null && fileSystemUtils.DirectoryExists(configuration.SbomDir.Value))
+        {
+            inputDir = configuration.SbomDir.Value;
+        }
+        else if (configuration.SbomPath?.Value != null && fileSystemUtils.FileExists(configuration.SbomPath.Value))
+        {
+            inputDir = fileSystemUtils.GetDirectoryName(configuration.SbomPath.Value);
+        }
+        else
+        {
+            throw new ArgumentException("No valid input SBOMs to redact provided.");
+        }
+
+        var outputDir = configuration.OutputPath.Value;
+        if (fileSystemUtils.GetFullPath(outputDir).Equals(fileSystemUtils.GetFullPath(inputDir)))
+        {
+            throw new ArgumentException("Output path cannot be the same as input SBOM directory.");
+        }
+
+        if (!fileSystemUtils.DirectoryExists(outputDir))
+        {
+            fileSystemUtils.CreateDirectory(outputDir);
+        }
+
+        foreach (var sbom in GetInputSbomPaths())
+        {
+            var outputPath = GetOutputPath(sbom);
+            if (fileSystemUtils.FileExists(outputPath))
+            {
+                throw new ArgumentException($"Output file {outputPath} already exists. Please update and try again.");
+            }
+        }
+
+        return outputDir;
     }
 }

src/Microsoft.Sbom.Common/FileSystemUtils.cs

@@ -67,6 +67,9 @@ public string JoinPaths(string root, string relativePath, string secondRelativeP
     /// <inheritdoc />
     public bool FileExists(string path) => File.Exists(path);
 
+    /// <inheritdoc />
+    public string GetFileName(string filePath) => Path.GetFileName(filePath);
+
     /// <inheritdoc />
     public Stream OpenWrite(string filePath) => new FileStream(
         filePath,

src/Microsoft.Sbom.Common/IFileSystemUtils.cs

@@ -104,6 +104,13 @@ public interface IFileSystemUtils
     /// <returns>True if the file exists, false otherwise.</returns>
     bool FileExists(string path);
 
+    /// <summary>
+    /// Get the file name of a file.
+    /// </summary>
+    /// <param name="filePath">The absolute path of the file.</param>
+    /// <returns>The file name.</returns>
+    string GetFileName(string filePath);
+
     /// <summary>
     /// Get the directory name of a file.
     /// </summary>

src/Microsoft.Sbom.Extensions.DependencyInjection/ServiceCollectionExtensions.cs

@@ -15,6 +15,7 @@
 using Microsoft.Sbom.Api.Entities.Output;
 using Microsoft.Sbom.Api.Executors;
 using Microsoft.Sbom.Api.Filters;
+using Microsoft.Sbom.Api.FormatValidator;
 using Microsoft.Sbom.Api.Hashing;
 using Microsoft.Sbom.Api.Manifest;
 using Microsoft.Sbom.Api.Manifest.Configuration;
@@ -73,6 +74,8 @@ public static IServiceCollection AddSbomTool(this IServiceCollection services, L
             .AddTransient<IWorkflow<SbomParserBasedValidationWorkflow>, SbomParserBasedValidationWorkflow>()
             .AddTransient<IWorkflow<SbomGenerationWorkflow>, SbomGenerationWorkflow>()
             .AddTransient<IWorkflow<SbomRedactionWorkflow>, SbomRedactionWorkflow>()
+            .AddTransient<ISbomRedactor, SbomRedactor>()
+            .AddTransient<ValidatedSBOMFactory>()
             .AddTransient<DirectoryWalker>()
             .AddTransient<IFilter<DownloadedRootPathFilter>, DownloadedRootPathFilter>()
             .AddTransient<IFilter<ManifestFolderFilter>, ManifestFolderFilter>()

src/Microsoft.Sbom.Parsers.Spdx22SbomParser/Entities/CreationInfo.cs

@@ -11,6 +11,7 @@ namespace Microsoft.Sbom.Parsers.Spdx22SbomParser.Entities;
 /// </summary>
 public class CreationInfo
 {
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     [JsonPropertyName("comment")]
     public string Comment { get; set; }
 
@@ -27,6 +28,7 @@ public class CreationInfo
     [JsonPropertyName("creators")]
     public IEnumerable<string> Creators { get; set; }
 
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     [JsonPropertyName("licenseListVersion")]
     public string LicenseListVersion { get; set; }
 }

src/Microsoft.Sbom.Parsers.Spdx22SbomParser/Entities/FormatEnforcedSPDX2.cs

@@ -10,12 +10,14 @@ public class FormatEnforcedSPDX2 : SPDX2RequiredProperties
 {
     // These attributes are not required by the SPDX spec, but may be present in
     // SBOMs produced by sbom-tool or 3P SBOMs. We want to (de)serialize them if they are present.
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     [JsonPropertyName("comment")]
     public string Comment { get; set; }
 
     [JsonPropertyName("documentDescribes")]
     public IEnumerable<string> DocumentDescribes { get; set; }
 
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     [JsonPropertyName("files")]
     public IEnumerable<SPDXFile> Files { get; set; }