Spdx 3.0 Parser for SBOM files (#860)

* Parsing spdx 3.0 with Unit tests and support for parsing with different compliance standards

* Fix unit test syntax issues, clean up json parser

* remove unnecessary changes, clean up assemblies

* fix bugs in generator tests

* addressing pr comments

* remove unused param

* make sure compliance standard has error handling and use enums

---------

Co-authored-by: ppandrate <ppandrate@microsoft.com>

Author: pragnya17

src/Microsoft.Sbom.Parsers.Spdx22SbomParser/JsonAsynchronousNodeKit/Constants.cs renamed to src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/Constants.cs

@@ -21,7 +21,7 @@ namespace Microsoft.Sbom.JsonAsynchronousNodeKit;
 /// This class is not Thread-safe since the stream and JsonReaders assume a single forward-only reader.
 /// Because of the use of recursion in the GetObject method, this class is also not suitable for parsing very deep json objects.
 /// </remarks>
-internal class LargeJsonParser
+public class LargeJsonParser
 {
     private const int DefaultReadBufferSize = 4096;
     private readonly Stream stream;
@@ -43,7 +43,6 @@ public LargeJsonParser(
         this.stream = stream ?? throw new ArgumentNullException(nameof(stream));
         this.handlers = handlers ?? throw new ArgumentNullException(nameof(handlers));
         this.jsonSerializerOptions = jsonSerializerOptions ?? new JsonSerializerOptions();
-
         this.buffer = new byte[bufferSize];
 
         // Validate buffer is not of 0 length.
@@ -153,6 +152,7 @@ private ParserStateResult HandleExplicitProperty(ref Utf8JsonReader reader, stri
         var handler = this.handlers![propertyName];
 
         object? result;
+
         switch (handler.Type)
         {
             case ParameterType.String:
@@ -268,11 +268,23 @@ private IEnumerable<object> GetArray(Type type)
 
     private object GetObject(Type type, ref Utf8JsonReader reader)
     {
-        var jsonObject = ParserUtils.ParseObject(this.stream, ref this.buffer, ref reader);
-        object? result = jsonObject;
-        if (type != typeof(JsonNode))
+        object? result = null;
+
+        if (type == typeof(string))
+        {
+            result = reader.GetString();
+        }
+        else
         {
-            result = jsonObject.Deserialize(type, this.jsonSerializerOptions);
+            var jsonObject = ParserUtils.ParseObject(this.stream, ref this.buffer, ref reader);
+            if (type != typeof(JsonNode))
+            {
+                result = jsonObject.Deserialize(type, this.jsonSerializerOptions);
+            }
+            else
+            {
+                result = jsonObject;
+            }
         }
 
         if (result is null)

src/Microsoft.Sbom.Parsers.Spdx22SbomParser/JsonAsynchronousNodeKit/Exceptions/ParserException.cs renamed to src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/Exceptions/ParserException.cs

@@ -9,7 +9,7 @@ namespace Microsoft.Sbom.Contracts;
 /// <summary>
 /// The object representation of all the metadata in an SPDX document.
 /// </summary>
-public class Spdx22Metadata
+public class SpdxMetadata
 {
     /// <summary>
     /// The version of the SPDX specification used in this document.

src/Microsoft.Sbom.Parsers.Spdx22SbomParser/JsonAsynchronousNodeKit/LargeJsonParser.cs renamed to src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/LargeJsonParser.cs

@@ -27,7 +27,7 @@ public interface ISbomParser
     /// </summary>
     /// <param name="stream"></param>
     /// <returns></returns>
-    Spdx22Metadata GetMetadata();
+    SpdxMetadata GetMetadata();
 
     /// <summary>
     /// This function is called by the sbom tool upon initialization to get all the

src/Microsoft.Sbom.Parsers.Spdx22SbomParser/JsonAsynchronousNodeKit/ParameterType.cs renamed to src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/ParameterType.cs

@@ -172,14 +172,14 @@ internal SPDXParser(
         return null;
     }
 
-    public Spdx22Metadata GetMetadata()
+    public SpdxMetadata GetMetadata()
     {
         if (!this.parsingComplete)
         {
             throw new ParserException($"{nameof(this.GetMetadata)} can only be called after Parsing is complete to ensure that a whole object is returned.");
         }
 
-        var spdxMetadata = new Spdx22Metadata();
+        var spdxMetadata = new SpdxMetadata();
         foreach (var kvp in this.metadata)
         {
             switch (kvp.Key)

src/Microsoft.Sbom.Parsers.Spdx22SbomParser/JsonAsynchronousNodeKit/ParserUtils.cs renamed to src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/ParserUtils.cs

@@ -16,6 +16,9 @@ internal static class Constants
     internal const string SPDXDocumentNameFormatString = "{0} {1}";
     internal const string PackageSupplierFormatString = "Organization: {0}";
 
+    internal const string SPDXContextHeaderName = "@context";
+    internal const string SPDXGraphHeaderName = "@graph";
+
     /// <summary>
     /// Use if SPDX creator
     /// - made an attempt to retrieve the info but cannot determine correct values.
@@ -24,6 +27,8 @@ internal static class Constants
     /// </summary>
     internal const string NoAssertionValue = "NOASSERTION";
 
+    internal const int ReadBufferSize = 4096;
+
     /// <summary>
     /// The <see cref="NoAssertionValue"/> value as a list with a single item.
     /// </summary>

src/Microsoft.Sbom.Parsers.Spdx22SbomParser/JsonAsynchronousNodeKit/PropertyHandler.cs renamed to src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/PropertyHandler.cs

@@ -45,7 +45,7 @@ protected Element()
 
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     [JsonPropertyName("name")]
-    public string Name { get; set; }
+    public virtual string Name { get; set; }
 
     /// <summary>
     /// Gets or sets unique Identifier for elements in SPDX document.
@@ -63,7 +63,7 @@ protected Element()
     /// </summary>
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
     [JsonPropertyName("verifiedUsing")]
-    public List<PackageVerificationCode> VerifiedUsing { get; set; }
+    public virtual List<PackageVerificationCode> VerifiedUsing { get; set; }
 
     [JsonRequired]
     [JsonPropertyName("type")]

src/Microsoft.Sbom.Contracts/Contracts/Spdx22Metadata.cs renamed to src/Microsoft.Sbom.Contracts/Contracts/SpdxMetadata.cs

@@ -0,0 +1,17 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System.Diagnostics.CodeAnalysis;
+using System.Text.Json.Serialization;
+
+namespace Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities.Enums;
+
+/// <summary>
+/// Defines the different supported compliance standards.
+/// </summary>
+[JsonConverter(typeof(JsonStringEnumConverter))]
+public enum ComplianceStandard
+{
+    NTIA,
+    None,
+}

src/Microsoft.Sbom.Extensions/ISbomParser.cs

@@ -22,4 +22,8 @@ public File()
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     [JsonPropertyName("mediaType")]
     public object MediaType { get; set; }
+
+    [JsonRequired]
+    [JsonPropertyName("name")]
+    public override string Name { get; set; }
 }

src/Microsoft.Sbom.Parsers.Spdx22SbomParser/Parser/SPDXParser.cs

@@ -25,7 +25,7 @@ public class FormatEnforcedSPDX3
     /// </summary>
     [JsonRequired]
     [JsonPropertyName("@graph")]
-    public List<Element> Graph { get; set; }
+    public IEnumerable<Element> Graph { get; set; }
 
     /// <summary>
     /// Required as part of the Core profile.

src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Constants.cs

@@ -0,0 +1,33 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System.Collections;
+using System.Collections.Generic;
+using System.Text.Json.Serialization;
+
+namespace Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities;
+
+/// <summary>
+/// Refers to any object that stores content on a computer.
+/// The type of content can optionally be provided in the contentType property.
+/// https://spdx.github.io/spdx-spec/v3.0.1/model/Software/Classes/File/
+/// An NTIA file specifically describes a file compliant with the NTIA SBOM standard.
+/// </summary>
+public class NTIAFile : File
+{
+    public NTIAFile()
+    {
+        Type = "software_File";
+    }
+
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    [JsonPropertyName("software_contentType")]
+    public object ContentType { get; set; }
+
+    /// <summary>
+    /// Make verification code required for Files. This is an internal requirement, not a requirement from SPDX.
+    /// </summary>
+    [JsonRequired]
+    [JsonPropertyName("verifiedUsing")]
+    public override List<PackageVerificationCode> VerifiedUsing { get; set; }
+}

src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/Element.cs

@@ -0,0 +1,25 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System.Collections.Generic;
+using System.Text.Json.Serialization;
+using Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities.Enums;
+
+namespace Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities;
+
+/// <summary>
+/// The SpdxDocument provides a convenient way to express information about collections of SPDX Elements that could potentially be serialized as complete units (e.g., all in-scope SPDX data within a single JSON-LD file).
+/// https://spdx.github.io/spdx-spec/v3.0.1/model/Core/Classes/SpdxDocument/
+/// An NTIA SpdxDocument specifically describes a SpdxDocument entity compliant with the NTIA SBOM standard.
+/// </summary>
+public class NTIASpdxDocument : SpdxDocument
+{
+    public NTIASpdxDocument()
+    {
+        Type = nameof(SpdxDocument);
+    }
+
+    [JsonRequired]
+    [JsonPropertyName("name")]
+    public override string Name { get; set; }
+}

src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/Enums/ComplianceStandard.cs

@@ -22,5 +22,9 @@ public Package()
     /// </summary>
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     [JsonPropertyName("suppliedBy")]
-    public string SuppliedBy { get; set; }
+    public virtual string SuppliedBy { get; set; }
+
+    [JsonRequired]
+    [JsonPropertyName("name")]
+    public override string Name { get; set; }
 }

src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/File.cs

@@ -13,12 +13,12 @@ namespace Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities;
 /// Defines relationships between elements in the current SBOM.
 /// https://spdx.github.io/spdx-spec/v3.0.1/model/Core/Classes/Relationship/
 /// </summary>
-public class Spdx30Relationship : Element
+public class Relationship : Element
 {
     /// <summary>
-    /// Initializes a new instance of the <see cref="Spdx30Relationship"/> class.
+    /// Initializes a new instance of the <see cref="Relationship"/> class.
     /// </summary>
-    public Spdx30Relationship()
+    public Relationship()
     {
         Type = nameof(Relationship);
     }

src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/FormatEnforcedSPDX3.cs

@@ -4,8 +4,6 @@
 using System.Collections.Generic;
 using System.Text.Json.Serialization;
 using Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities;
-using Newtonsoft.Json;
-using JsonIgnoreAttribute = System.Text.Json.Serialization.JsonIgnoreAttribute;
 
 /// <summary>
 /// Class defined as specified in: https://spdx.github.io/spdx-spec/v3.0.1/model/Software/Software/
@@ -37,8 +35,8 @@ public abstract class Software : Element
     public string ContentIdentifierValue { get; set; }
 
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-    [JsonProperty(PropertyName = "software_copyrightText")]
-    public string CopyrightText { get; set; }
+    [JsonPropertyName("software_copyrightText")]
+    public virtual string CopyrightText { get; set; }
 
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     [JsonPropertyName("software_downloadLocation")]
@@ -62,7 +60,7 @@ public abstract class Software : Element
 
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     [JsonPropertyName("software_packageVersion")]
-    public string PackageVersion { get; set; }
+    public virtual string PackageVersion { get; set; }
 
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     [JsonPropertyName("software_primaryPurpose")]
@@ -77,6 +75,6 @@ public abstract class Software : Element
     public File SnippetFromFile { get; set; }
 
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-    [JsonPropertyName("software_copyrightText")]
+    [JsonPropertyName("software_sourceInfo")]
     public string SourceInfo { get; set; }
 }

src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/NTIAFile.cs

@@ -23,7 +23,7 @@ public class SpdxDocument : Element
 
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     [JsonPropertyName("namespaceMap")]
-    public List<NamespaceMap> NamespaceMap { get; set; }
+    public Dictionary<string, string> NamespaceMap { get; set; }
 
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     [JsonPropertyName("element")]
@@ -35,5 +35,5 @@ public class SpdxDocument : Element
 
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     [JsonPropertyName("rootElement")]
-    public List<Element> RootElement { get; set; }
+    public List<string> RootElement { get; set; }
 }