From 9b96da327ee3aa9607271e7cb38391a5365b6ebf Mon Sep 17 00:00:00 2001 From: ppandrate Date: Thu, 9 Jan 2025 18:23:08 -0800 Subject: [PATCH 1/4] Parsing spdx 3.0 with Unit tests and support for parsing with different compliance standards --- .../JsonAsynchronousNodeKit/Constants.cs | 0 .../Exceptions/ParserException.cs | 0 .../LargeJsonParser.cs | 2 +- .../LargeJsonParser30.cs | 304 ++++++++++++ .../JsonAsynchronousNodeKit/ParameterType.cs | 0 .../JsonAsynchronousNodeKit/ParserUtils.cs | 21 + .../PropertyHandler.cs | 0 .../Microsoft.Sbom.Common.csproj | 1 + .../{Spdx22Metadata.cs => SpdxMetadata.cs} | 2 +- src/Microsoft.Sbom.Extensions/ISbomParser.cs | 2 +- .../Parser/SPDXParser.cs | 7 +- .../Constants.cs | 6 + .../Entities/Element.cs | 4 +- .../Entities/Enums/RelationshipType.cs | 16 +- .../Entities/File.cs | 4 + .../Entities/FormatEnforcedSPDX3.cs | 2 +- .../Entities/NTIAFile.cs | 33 ++ .../Entities/NTIASpdxDocument.cs | 25 + .../Entities/NamespaceMap.cs | 16 - .../Entities/Package.cs | 6 +- ...{Spdx30Relationship.cs => Relationship.cs} | 6 +- .../Entities/Software.cs | 10 +- .../Entities/SpdxDocument.cs | 4 +- .../Generator.cs | 28 +- .../Parser/ContextsResult.cs | 19 + .../Parser/ElementsResult.cs | 19 + .../Parser/ExternalMapsResult.cs | 19 + .../Parser/FilesResult.cs | 19 + .../Parser/PackagesResult.cs | 19 + .../Parser/ParserResults.cs | 17 + .../Parser/RelationshipsResult.cs | 19 + .../Parser/SPDX30Parser.cs | 156 ++++++ .../Utils/ElementSerializer.cs | 5 +- .../Utils/SPDXExtensions.cs | 7 +- .../Validator.cs | 31 ++ .../SbomFullDocWithFilesStrings.cs | 459 ++++++++++++++++++ .../SbomFullDocWithMetadataJsonStrings.cs | 197 ++++++++ .../Parser/SbomFileParserTests.cs | 60 +++ .../Parser/SbomMetadataParserTests.cs | 162 +++++++ .../Parser/SbomParserTestsBase.cs | 311 ++++++++++++ 40 files changed, 1948 insertions(+), 70 deletions(-) rename src/{Microsoft.Sbom.Parsers.Spdx22SbomParser => Microsoft.Sbom.Common}/JsonAsynchronousNodeKit/Constants.cs (100%) rename src/{Microsoft.Sbom.Parsers.Spdx22SbomParser => Microsoft.Sbom.Common}/JsonAsynchronousNodeKit/Exceptions/ParserException.cs (100%) rename src/{Microsoft.Sbom.Parsers.Spdx22SbomParser => Microsoft.Sbom.Common}/JsonAsynchronousNodeKit/LargeJsonParser.cs (99%) create mode 100644 src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/LargeJsonParser30.cs rename src/{Microsoft.Sbom.Parsers.Spdx22SbomParser => Microsoft.Sbom.Common}/JsonAsynchronousNodeKit/ParameterType.cs (100%) rename src/{Microsoft.Sbom.Parsers.Spdx22SbomParser => Microsoft.Sbom.Common}/JsonAsynchronousNodeKit/ParserUtils.cs (92%) rename src/{Microsoft.Sbom.Parsers.Spdx22SbomParser => Microsoft.Sbom.Common}/JsonAsynchronousNodeKit/PropertyHandler.cs (100%) rename src/Microsoft.Sbom.Contracts/Contracts/{Spdx22Metadata.cs => SpdxMetadata.cs} (97%) create mode 100644 src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/NTIAFile.cs create mode 100644 src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/NTIASpdxDocument.cs delete mode 100644 src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/NamespaceMap.cs rename src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/{Spdx30Relationship.cs => Relationship.cs} (91%) create mode 100644 src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/ContextsResult.cs create mode 100644 src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/ElementsResult.cs create mode 100644 src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/ExternalMapsResult.cs create mode 100644 src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/FilesResult.cs create mode 100644 src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/PackagesResult.cs create mode 100644 src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/ParserResults.cs create mode 100644 src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/RelationshipsResult.cs create mode 100644 src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs create mode 100644 src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Validator.cs create mode 100644 test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/JsonStrings/SbomFullDocWithFilesStrings.cs create mode 100644 test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/JsonStrings/SbomFullDocWithMetadataJsonStrings.cs create mode 100644 test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomFileParserTests.cs create mode 100644 test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomMetadataParserTests.cs create mode 100644 test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomParserTestsBase.cs diff --git a/src/Microsoft.Sbom.Parsers.Spdx22SbomParser/JsonAsynchronousNodeKit/Constants.cs b/src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/Constants.cs similarity index 100% rename from src/Microsoft.Sbom.Parsers.Spdx22SbomParser/JsonAsynchronousNodeKit/Constants.cs rename to src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/Constants.cs diff --git a/src/Microsoft.Sbom.Parsers.Spdx22SbomParser/JsonAsynchronousNodeKit/Exceptions/ParserException.cs b/src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/Exceptions/ParserException.cs similarity index 100% rename from src/Microsoft.Sbom.Parsers.Spdx22SbomParser/JsonAsynchronousNodeKit/Exceptions/ParserException.cs rename to src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/Exceptions/ParserException.cs diff --git a/src/Microsoft.Sbom.Parsers.Spdx22SbomParser/JsonAsynchronousNodeKit/LargeJsonParser.cs b/src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/LargeJsonParser.cs similarity index 99% rename from src/Microsoft.Sbom.Parsers.Spdx22SbomParser/JsonAsynchronousNodeKit/LargeJsonParser.cs rename to src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/LargeJsonParser.cs index 4815b0f8..a96de0f0 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx22SbomParser/JsonAsynchronousNodeKit/LargeJsonParser.cs +++ b/src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/LargeJsonParser.cs @@ -21,7 +21,7 @@ namespace Microsoft.Sbom.JsonAsynchronousNodeKit; /// This class is not Thread-safe since the stream and JsonReaders assume a single forward-only reader. /// Because of the use of recursion in the GetObject method, this class is also not suitable for parsing very deep json objects. /// -internal class LargeJsonParser +public class LargeJsonParser { private const int DefaultReadBufferSize = 4096; private readonly Stream stream; diff --git a/src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/LargeJsonParser30.cs b/src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/LargeJsonParser30.cs new file mode 100644 index 00000000..dcd3677d --- /dev/null +++ b/src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/LargeJsonParser30.cs @@ -0,0 +1,304 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Linq.Expressions; +using System.Net.Http.Json; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; +using System.Xml.Linq; +using Microsoft.Sbom.JsonAsynchronousNodeKit.Exceptions; +using Newtonsoft.Json; +using JsonException = System.Text.Json.JsonException; +using JsonSerializer = System.Text.Json.JsonSerializer; + +namespace Microsoft.Sbom.JsonAsynchronousNodeKit; + +#nullable enable + +/// +/// Allows for parsing large json objects without loading the entire object into memory. Large json arrays use a yield return to avoid having the whole enumerable in memory at once. +/// +/// +/// This class is not Thread-safe since the stream and JsonReaders assume a single forward-only reader. +/// Because of the use of recursion in the GetObject method, this class is also not suitable for parsing very deep json objects. +/// +public class LargeJsonParser30 +{ + private const int DefaultReadBufferSize = 4096; + private readonly Stream stream; + private readonly JsonSerializerOptions jsonSerializerOptions; + private byte[] buffer; + private JsonReaderState readerState; + private bool isFinalBlock; + private bool isParsingStarted = false; + private bool enumeratorActive = false; + private ParserStateResult? previousResultState = null; + private readonly Dictionary handlers; + + public LargeJsonParser30( + Stream stream, + Dictionary handlers, + JsonSerializerOptions? jsonSerializerOptions = default, + int bufferSize = DefaultReadBufferSize) + { + this.stream = stream ?? throw new ArgumentNullException(nameof(stream)); + this.handlers = handlers ?? throw new ArgumentNullException(nameof(handlers)); + this.jsonSerializerOptions = jsonSerializerOptions ?? new JsonSerializerOptions(); + + this.buffer = new byte[bufferSize]; + + // Validate buffer is not of 0 length. + if (this.buffer.Length == 0) + { + throw new ArgumentException($"The {nameof(this.buffer)} value can't be null or of 0 length."); + } + + this.stream.Position = GetStartPosition(this.stream); + + if (!stream.CanRead) + { + throw new NotSupportedException("The stream must be readable."); + } + + if (stream.Read(this.buffer) == 0) + { + throw new EndOfStreamException("No bytes were read from the stream."); + } + + static int GetStartPosition(Stream stream) + { + var bom = Encoding.UTF8.Preamble.ToArray(); + stream.Position = 0; + var buffer = new byte[bom.Length]; + _ = stream.Read(buffer, 0, buffer.Length); + + return Enumerable.SequenceEqual(buffer, bom) ? 3 : 0; + } + } + + /// + /// Begin evaluating the next section of the json stream. + /// + /// A ParserStateResult object describing the section that was encountered or null if this was the final state. + /// If the result object is an enumerable you MUST ensure that you've walked it entirely before calling next again. + public ParserStateResult? Next() + { + if (this.enumeratorActive) + { + throw new ParserException("You must walk the entire enumerable from the previous result before calling Next() again."); + } + + try + { + var reader = new Utf8JsonReader(this.buffer, isFinalBlock: this.isFinalBlock, this.readerState); + + if (!this.isParsingStarted) + { + ParserUtils.SkipNoneTokens(this.stream, ref this.buffer, ref reader); + + // Arrays are legal root objects, but if you expect that you should use `System.Text.Json.JsonSerializer.DeserializeAsyncEnumerable` instead. + if (reader.TokenType == JsonTokenType.StartArray) + { + throw new ParserException($"For root-level arrays use {nameof(JsonSerializer.DeserializeAsyncEnumerable)}."); + } + + ParserUtils.AssertTokenType(this.stream, ref reader, JsonTokenType.StartObject); + ParserUtils.GetMoreBytesFromStream(this.stream, ref this.buffer, ref reader); + + this.isParsingStarted = true; + } + + ParserUtils.Read(this.stream, ref this.buffer, ref reader); + + // If the end of the Json Object is reached, return null to indicate completion. + if (reader.TokenType == JsonTokenType.EndObject) + { + return null; + } + else if (this.previousResultState is not null && this.previousResultState.YieldReturn && reader.TokenType == JsonTokenType.EndArray) + { + // yield returning json arrays means we can't pass it the same Utf8JsonReader ref, so we need to create a new one. + // BUT when we do that we end up consuming the next token, so we need to leave it in the array case to be eaten by the next caller. + ParserUtils.Read(this.stream, ref this.buffer, ref reader); + } + + ParserUtils.AssertTokenType(this.stream, ref reader, JsonTokenType.PropertyName); + + var propertyName = reader.GetString() ?? throw new InvalidOperationException("It should not be possible to have a null PropertyName"); + ParserUtils.Read(this.stream, ref this.buffer, ref reader); + + var resultState = this.handlers.ContainsKey(propertyName) + ? this.HandleExplicitProperty(ref reader, propertyName) + : this.HandleExtraProperty(ref reader, propertyName); + + // yield return is a special case where we need to leave the reader in the same state as we found it. + if (!resultState.YieldReturn) + { + ParserUtils.GetMoreBytesFromStream(this.stream, ref this.buffer, ref reader); + + this.isFinalBlock = reader.IsFinalBlock; + this.readerState = reader.CurrentState; + } + + this.previousResultState = resultState; + + return resultState; + } + catch (JsonException ex) + { + throw new ParserException($"Error parsing json at position {this.stream.Position}", ex); + } + } + + private ParserStateResult HandleExplicitProperty(ref Utf8JsonReader reader, string propertyName) + { + var handler = this.handlers![propertyName]; + + object? result; + switch (handler.Type) + { + case ParameterType.String: + ParserUtils.AssertTokenType(this.stream, ref reader, JsonTokenType.String); + result = reader.GetString(); + break; + case ParameterType.Skip: + ParserUtils.SkipProperty(this.stream, ref this.buffer, ref reader); + result = null; + break; + case ParameterType.Int: + ParserUtils.AssertTokenType(this.stream, ref reader, JsonTokenType.Number); + var i = reader.GetInt32(); + result = i; + break; + case ParameterType.Object: + var objType = handler.GetType().GetGenericArguments()[0]; + result = this.GetObject(ref reader, objType); + break; + case ParameterType.Array: + var arrType = handler.GetType().GetGenericArguments()[0]; + result = this.ParseArray(ref reader, arrType); + break; + default: + throw new InvalidOperationException($"Unknown {nameof(ParameterType)}: {handler.Type}"); + } + + return new ParserStateResult(propertyName, result, ExplicitField: true, YieldReturn: handler.Type == ParameterType.Array); + } + + private ParserStateResult HandleExtraProperty(ref Utf8JsonReader reader, string propertyName) + { + object? result = reader.TokenType switch + { + JsonTokenType.String => reader.GetString(), + JsonTokenType.Number => reader.GetInt32(), + JsonTokenType.True => true, + JsonTokenType.False => false, + JsonTokenType.StartArray => ParserUtils.ParseArray(this.stream, ref this.buffer, ref reader), + JsonTokenType.StartObject => ParserUtils.ParseObject(this.stream, ref this.buffer, ref reader), + JsonTokenType.None => throw new NotImplementedException(), + JsonTokenType.EndObject => throw new NotImplementedException(), + JsonTokenType.EndArray => throw new NotImplementedException(), + JsonTokenType.PropertyName => throw new NotImplementedException(), + JsonTokenType.Comment => throw new NotImplementedException(), + JsonTokenType.Null => throw new NotImplementedException(), + _ => throw new InvalidOperationException($"Unknown {nameof(JsonTokenType)}: {reader.TokenType}"), + }; + return new ParserStateResult(propertyName, result, ExplicitField: false, YieldReturn: false); + } + + private IEnumerable ParseArray(ref Utf8JsonReader reader, Type objType) + { + ParserUtils.AssertTokenType(this.stream, ref reader, JsonTokenType.StartArray); + + // We can't pass the current reader along, so we need to save it's state for it. + ParserUtils.GetMoreBytesFromStream(this.stream, ref this.buffer, ref reader); + + this.isFinalBlock = reader.IsFinalBlock; + this.readerState = reader.CurrentState; + + return this.GetArray(objType); + } + + private IEnumerable GetArray(Type type) + { + this.enumeratorActive = true; + while (true) + { + var obj = this.ReadArrayObject(type); + + if (obj is not null) + { + yield return obj; + } + else + { + this.enumeratorActive = false; + yield break; + } + } + } + + private object? ReadArrayObject(Type type) + { + try + { + var reader = new Utf8JsonReader(this.buffer, this.isFinalBlock, this.readerState); + + ParserUtils.Read(this.stream, ref this.buffer, ref reader); + + object? result; + if (reader.TokenType == JsonTokenType.EndArray) + { + result = null; + } + else if (type.Name == "String") + { + result = reader.GetString(); + } + else + { + result = this.GetObject(ref reader, type); + } + + ParserUtils.GetMoreBytesFromStream(this.stream, ref this.buffer, ref reader); + + this.isFinalBlock = reader.IsFinalBlock; + this.readerState = reader.CurrentState; + + return result; + } + catch (JsonException ex) + { + throw new ParserException($"Error parsing json at position {this.stream.Position}", ex); + } + } + + private object GetObject(ref Utf8JsonReader reader, Type type) + { + var jsonObject = ParserUtils.ParseObject(this.stream, ref this.buffer, ref reader); + var jsonObjectType = jsonObject.AsObject()["type"]?.ToString(); + object? result; + + if (jsonObjectType == null) + { + throw new InvalidOperationException($"Deserializing {jsonObjectType} object type is not supported."); + } + else + { + result = JsonConvert.DeserializeObject(jsonObject.ToString()); + } + + if (result is null) + { + throw new InvalidOperationException($"Deserialization unexpectedly returned null."); + } + + return result; + } +} diff --git a/src/Microsoft.Sbom.Parsers.Spdx22SbomParser/JsonAsynchronousNodeKit/ParameterType.cs b/src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/ParameterType.cs similarity index 100% rename from src/Microsoft.Sbom.Parsers.Spdx22SbomParser/JsonAsynchronousNodeKit/ParameterType.cs rename to src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/ParameterType.cs diff --git a/src/Microsoft.Sbom.Parsers.Spdx22SbomParser/JsonAsynchronousNodeKit/ParserUtils.cs b/src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/ParserUtils.cs similarity index 92% rename from src/Microsoft.Sbom.Parsers.Spdx22SbomParser/JsonAsynchronousNodeKit/ParserUtils.cs rename to src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/ParserUtils.cs index bb76e0b7..b13eafce 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx22SbomParser/JsonAsynchronousNodeKit/ParserUtils.cs +++ b/src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/ParserUtils.cs @@ -3,6 +3,7 @@ using System; using System.IO; +using System.Reflection.Metadata.Ecma335; using System.Text.Json; using System.Text.Json.Nodes; using Microsoft.Sbom.JsonAsynchronousNodeKit.Exceptions; @@ -209,6 +210,26 @@ internal static JsonObject ParseObject(Stream stream, ref byte[] buffer, ref Utf return node; } + internal static JsonNode ParseObjectAsString(Stream stream, ref byte[] buffer, ref Utf8JsonReader reader) + { + AssertTokenType(stream, ref reader, JsonTokenType.StartObject); + + JsonNode? value = null; + while (reader.TokenType != JsonTokenType.EndObject || value is JsonObject) + { + Read(stream, ref buffer, ref reader); + if (reader.TokenType == JsonTokenType.EndObject) + { + break; + } + + Read(stream, ref buffer, ref reader); + value = ParseValue(stream, ref buffer, ref reader); + } + + return value ?? throw new InvalidOperationException("Parsed value as string cannot be null."); + } + internal static JsonNode? ParseValue(Stream stream, ref byte[] buffer, ref Utf8JsonReader reader) => reader.TokenType switch { JsonTokenType.StartObject => ParseObject(stream, ref buffer, ref reader), diff --git a/src/Microsoft.Sbom.Parsers.Spdx22SbomParser/JsonAsynchronousNodeKit/PropertyHandler.cs b/src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/PropertyHandler.cs similarity index 100% rename from src/Microsoft.Sbom.Parsers.Spdx22SbomParser/JsonAsynchronousNodeKit/PropertyHandler.cs rename to src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/PropertyHandler.cs diff --git a/src/Microsoft.Sbom.Common/Microsoft.Sbom.Common.csproj b/src/Microsoft.Sbom.Common/Microsoft.Sbom.Common.csproj index 0c4e6b62..174b0365 100644 --- a/src/Microsoft.Sbom.Common/Microsoft.Sbom.Common.csproj +++ b/src/Microsoft.Sbom.Common/Microsoft.Sbom.Common.csproj @@ -9,6 +9,7 @@ + diff --git a/src/Microsoft.Sbom.Contracts/Contracts/Spdx22Metadata.cs b/src/Microsoft.Sbom.Contracts/Contracts/SpdxMetadata.cs similarity index 97% rename from src/Microsoft.Sbom.Contracts/Contracts/Spdx22Metadata.cs rename to src/Microsoft.Sbom.Contracts/Contracts/SpdxMetadata.cs index b627c727..51985296 100644 --- a/src/Microsoft.Sbom.Contracts/Contracts/Spdx22Metadata.cs +++ b/src/Microsoft.Sbom.Contracts/Contracts/SpdxMetadata.cs @@ -9,7 +9,7 @@ namespace Microsoft.Sbom.Contracts; /// /// The object representation of all the metadata in an SPDX document. /// -public class Spdx22Metadata +public class SpdxMetadata { /// /// The version of the SPDX specification used in this document. diff --git a/src/Microsoft.Sbom.Extensions/ISbomParser.cs b/src/Microsoft.Sbom.Extensions/ISbomParser.cs index 243daa2e..8db84968 100644 --- a/src/Microsoft.Sbom.Extensions/ISbomParser.cs +++ b/src/Microsoft.Sbom.Extensions/ISbomParser.cs @@ -27,7 +27,7 @@ public interface ISbomParser /// /// /// - Spdx22Metadata GetMetadata(); + SpdxMetadata GetMetadata(); /// /// This function is called by the sbom tool upon initialization to get all the diff --git a/src/Microsoft.Sbom.Parsers.Spdx22SbomParser/Parser/SPDXParser.cs b/src/Microsoft.Sbom.Parsers.Spdx22SbomParser/Parser/SPDXParser.cs index 57c71771..b8f8747c 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx22SbomParser/Parser/SPDXParser.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx22SbomParser/Parser/SPDXParser.cs @@ -162,7 +162,8 @@ internal SPDXParser( { if (!this.observedFieldNames.Contains(requiredField)) { - throw new ParserException($"Required field {requiredField} was not found in the SPDX file"); + // TODO: remove this commented out part + // throw new ParserException($"Required field {requiredField} was not found in the SPDX file"); } } } @@ -172,14 +173,14 @@ internal SPDXParser( return null; } - public Spdx22Metadata GetMetadata() + public SpdxMetadata GetMetadata() { if (!this.parsingComplete) { throw new ParserException($"{nameof(this.GetMetadata)} can only be called after Parsing is complete to ensure that a whole object is returned."); } - var spdxMetadata = new Spdx22Metadata(); + var spdxMetadata = new SpdxMetadata(); foreach (var kvp in this.metadata) { switch (kvp.Key) diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Constants.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Constants.cs index 6b460cb6..8f3a3869 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Constants.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Constants.cs @@ -16,6 +16,10 @@ internal static class Constants internal const string SPDXDocumentNameFormatString = "{0} {1}"; internal const string PackageSupplierFormatString = "Organization: {0}"; + internal const string SPDXVersionHeaderName = "specVersion"; + internal const string SPDXContextHeaderName = "@context"; + internal const string SPDXGraphHeaderName = "@graph"; + /// /// Use if SPDX creator /// - made an attempt to retrieve the info but cannot determine correct values. @@ -24,6 +28,8 @@ internal static class Constants /// internal const string NoAssertionValue = "NOASSERTION"; + internal const int ReadBufferSize = 4096; + /// /// The value as a list with a single item. /// diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/Element.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/Element.cs index 8c33f930..4153a4ab 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/Element.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/Element.cs @@ -45,7 +45,7 @@ protected Element() [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] [JsonPropertyName("name")] - public string Name { get; set; } + public virtual string Name { get; set; } /// /// Gets or sets unique Identifier for elements in SPDX document. @@ -63,7 +63,7 @@ protected Element() /// [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] [JsonPropertyName("verifiedUsing")] - public List VerifiedUsing { get; set; } + public virtual List VerifiedUsing { get; set; } [JsonRequired] [JsonPropertyName("type")] diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/Enums/RelationshipType.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/Enums/RelationshipType.cs index ce212716..6a4e90b7 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/Enums/RelationshipType.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/Enums/RelationshipType.cs @@ -43,7 +43,7 @@ public enum RelationshipType CONTAINS, /// - /// The from Vulnerability is coordinatedBy the to Agent(s) (vendor, researcher, or consumer agent). + /// The from Vulnerability is coordinatedBy the to Agent(ContextsResult) (vendor, researcher, or consumer agent). /// COORDINATED_BY, @@ -69,7 +69,7 @@ public enum RelationshipType DESCENDANT_OF, /// - /// The from Element describes each to Element. To denote the root(s) of a tree of elements in a collection, the rootElement property should be used. + /// The from Element describes each to Element. To denote the root(ContextsResult) of a tree of elements in a collection, the rootElement property should be used. /// DESCRIBES, @@ -89,7 +89,7 @@ public enum RelationshipType EXPLOIT_CREATED_BY, /// - /// Designates a from Vulnerability has been fixed by the to Agent(s). + /// Designates a from Vulnerability has been fixed by the to Agent(ContextsResult). /// FIXED_BY, @@ -99,7 +99,7 @@ public enum RelationshipType FIXED_IN, /// - /// Designates a from Vulnerability was originally discovered by the to Agent(s). + /// Designates a from Vulnerability was originally discovered by the to Agent(ContextsResult). /// FOUND_BY, @@ -130,7 +130,7 @@ public enum RelationshipType /// /// The from Element treats each to Element as a data file. - /// A data file is an artifact that stores data required or optional for the from Element's functionality. + /// A data file is an artifact that stores data required or optional for the from Element'ContextsResult functionality. /// A data file can be a database file, an index file, a log file, an AI model file, a calibration data file, a temporary file, a backup file, and more. /// For AI training dataset, test dataset, test artifact, configuration data, build input data, and build output data, /// please consider using the more specific relationship types: trainedOn, testedOn, hasTest, configures, hasInput, and hasOutput, respectively. @@ -284,7 +284,7 @@ public enum RelationshipType REPORTED_BY, /// - /// Designates a from Vulnerability's details were tracked, aggregated, and/or enriched to improve context (i.e. NVD) by each to Agent. + /// Designates a from Vulnerability'ContextsResult details were tracked, aggregated, and/or enriched to improve context (i.e. NVD) by each to Agent. /// REPUBLISHED_BY, @@ -294,12 +294,12 @@ public enum RelationshipType SERIALIZED_IN_ARTIFACT, /// - /// The from Element has been tested on the to Element(s). + /// The from Element has been tested on the to Element(ContextsResult). /// TESTED_ON, /// - /// The from Element has been trained on the to Element(s). + /// The from Element has been trained on the to Element(ContextsResult). /// TRAINED_ON, diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/File.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/File.cs index 6b03da44..8667a151 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/File.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/File.cs @@ -22,4 +22,8 @@ public File() [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] [JsonPropertyName("mediaType")] public object MediaType { get; set; } + + [JsonRequired] + [JsonPropertyName("name")] + public override string Name { get; set; } } diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/FormatEnforcedSPDX3.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/FormatEnforcedSPDX3.cs index 3a7b1466..2168c5a4 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/FormatEnforcedSPDX3.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/FormatEnforcedSPDX3.cs @@ -25,7 +25,7 @@ public class FormatEnforcedSPDX3 /// [JsonRequired] [JsonPropertyName("@graph")] - public List Graph { get; set; } + public IEnumerable Graph { get; set; } /// /// Required as part of the Core profile. diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/NTIAFile.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/NTIAFile.cs new file mode 100644 index 00000000..44572c50 --- /dev/null +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/NTIAFile.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities; + +/// +/// Refers to any object that stores content on a computer. +/// The type of content can optionally be provided in the contentType property. +/// https://spdx.github.io/spdx-spec/v3.0.1/model/Software/Classes/File/ +/// An NTIA file specifically describes a file compliant with the NTIA SBOM standard. +/// +public class NTIAFile : File +{ + public NTIAFile() + { + Type = "software_File"; + } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("software_contentType")] + public object ContentType { get; set; } + + /// + /// Make verification code required for Files. This is an internal requirement, not a requirement from SPDX. + /// + [JsonRequired] + [JsonPropertyName("verifiedUsing")] + public override List VerifiedUsing { get; set; } +} diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/NTIASpdxDocument.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/NTIASpdxDocument.cs new file mode 100644 index 00000000..aa45654a --- /dev/null +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/NTIASpdxDocument.cs @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.Generic; +using System.Text.Json.Serialization; +using Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities.Enums; + +namespace Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities; + +/// +/// The SpdxDocument provides a convenient way to express information about collections of SPDX Elements that could potentially be serialized as complete units (e.g., all in-scope SPDX data within a single JSON-LD file). +/// https://spdx.github.io/spdx-spec/v3.0.1/model/Core/Classes/SpdxDocument/ +/// An NTIA SpdxDocument specifically describes a SpdxDocument entity compliant with the NTIA SBOM standard. +/// +public class NTIASpdxDocument : SpdxDocument +{ + public NTIASpdxDocument() + { + Type = nameof(SpdxDocument); + } + + [JsonRequired] + [JsonPropertyName("name")] + public override string Name { get; set; } +} diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/NamespaceMap.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/NamespaceMap.cs deleted file mode 100644 index eb53a324..00000000 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/NamespaceMap.cs +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -namespace Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities; - -using System.Text.Json.Serialization; - -/// -/// https://spdx.github.io/spdx-spec/v3.0.1/model/Core/Classes/NamespaceMap/ -/// -public class NamespaceMap : Element -{ - [JsonRequired] - [JsonPropertyName("namespace")] - public string Namespace { get; set; } -} diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/Package.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/Package.cs index a0a95f94..091df823 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/Package.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/Package.cs @@ -22,5 +22,9 @@ public Package() /// [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] [JsonPropertyName("suppliedBy")] - public string SuppliedBy { get; set; } + public virtual string SuppliedBy { get; set; } + + [JsonRequired] + [JsonPropertyName("name")] + public override string Name { get; set; } } diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/Spdx30Relationship.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/Relationship.cs similarity index 91% rename from src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/Spdx30Relationship.cs rename to src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/Relationship.cs index 974e8940..34309e10 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/Spdx30Relationship.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/Relationship.cs @@ -13,12 +13,12 @@ namespace Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities; /// Defines relationships between elements in the current SBOM. /// https://spdx.github.io/spdx-spec/v3.0.1/model/Core/Classes/Relationship/ /// -public class Spdx30Relationship : Element +public class Relationship : Element { /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// - public Spdx30Relationship() + public Relationship() { Type = nameof(Relationship); } diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/Software.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/Software.cs index a2e2ed0a..a2611bee 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/Software.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/Software.cs @@ -4,8 +4,6 @@ using System.Collections.Generic; using System.Text.Json.Serialization; using Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities; -using Newtonsoft.Json; -using JsonIgnoreAttribute = System.Text.Json.Serialization.JsonIgnoreAttribute; /// /// Class defined as specified in: https://spdx.github.io/spdx-spec/v3.0.1/model/Software/Software/ @@ -37,8 +35,8 @@ public abstract class Software : Element public string ContentIdentifierValue { get; set; } [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - [JsonProperty(PropertyName = "software_copyrightText")] - public string CopyrightText { get; set; } + [JsonPropertyName("software_copyrightText")] + public virtual string CopyrightText { get; set; } [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] [JsonPropertyName("software_downloadLocation")] @@ -62,7 +60,7 @@ public abstract class Software : Element [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] [JsonPropertyName("software_packageVersion")] - public string PackageVersion { get; set; } + public virtual string PackageVersion { get; set; } [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] [JsonPropertyName("software_primaryPurpose")] @@ -77,6 +75,6 @@ public abstract class Software : Element public File SnippetFromFile { get; set; } [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - [JsonPropertyName("software_copyrightText")] + [JsonPropertyName("software_sourceInfo")] public string SourceInfo { get; set; } } diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/SpdxDocument.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/SpdxDocument.cs index 04284f74..27537dd6 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/SpdxDocument.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/SpdxDocument.cs @@ -23,7 +23,7 @@ public class SpdxDocument : Element [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] [JsonPropertyName("namespaceMap")] - public List NamespaceMap { get; set; } + public Dictionary NamespaceMap { get; set; } [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] [JsonPropertyName("element")] @@ -35,5 +35,5 @@ public class SpdxDocument : Element [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] [JsonPropertyName("rootElement")] - public List RootElement { get; set; } + public List RootElement { get; set; } } diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Generator.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Generator.cs index 4b61b0cb..e91f3ef0 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Generator.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Generator.cs @@ -200,14 +200,14 @@ public GenerationResult GenerateRootPackage(IInternalMetadataProvider internalMe // Generate SPDX relationship elements to indicate no assertions are made about licenses for this root package. var noAssertionLicense = GenerateLicenseElement(null); - var spdxRelationshipLicenseDeclaredElement = new Spdx30Relationship + var spdxRelationshipLicenseDeclaredElement = new Entities.Relationship { From = spdxPackage.SpdxId, RelationshipType = RelationshipType.HAS_DECLARED_LICENSE, To = new List { noAssertionLicense.SpdxId }, }; - var spdxRelationshipLicenseConcludedElement = new Spdx30Relationship + var spdxRelationshipLicenseConcludedElement = new Entities.Relationship { From = spdxPackage.SpdxId, RelationshipType = RelationshipType.HAS_CONCLUDED_LICENSE, @@ -300,7 +300,7 @@ public GenerationResult GenerateJsonDocument(ExternalDocumentReferenceInfo exter /// Relationship info /// /// - public GenerationResult GenerateJsonDocument(Relationship relationship) + public GenerationResult GenerateJsonDocument(Extensions.Entities.Relationship relationship) { if (relationship is null) { @@ -314,7 +314,7 @@ public GenerationResult GenerateJsonDocument(Relationship relationship) : relationship.TargetElementId; var sourceElement = relationship.SourceElementId; - var spdxRelationship = new Spdx30Relationship + var spdxRelationship = new Entities.Relationship { From = sourceElement, RelationshipType = this.GetSPDXRelationshipType(relationship.RelationshipType), @@ -374,12 +374,8 @@ public GenerationResult GenerateJsonDocument(IInternalMetadataProvider internalM CreatedUsing = new List { spdxTool.SpdxId }, }; - var spdxNamespaceMap = new NamespaceMap - { - Namespace = internalMetadataProvider.GetDocumentNamespace(), - }; - - spdxNamespaceMap.AddSpdxId(); + var spdxNamespaceMap = new Dictionary(); + spdxNamespaceMap["sbom"] = internalMetadataProvider.GetDocumentNamespace(); var spdxDataLicense = new AnyLicenseInfo { @@ -390,7 +386,7 @@ public GenerationResult GenerateJsonDocument(IInternalMetadataProvider internalM var spdxDocument = new SpdxDocument { DataLicense = spdxDataLicense.SpdxId, - NamespaceMap = new List { spdxNamespaceMap }, + NamespaceMap = spdxNamespaceMap, SpdxId = Constants.SPDXDocumentIdValue, Name = documentName, ProfileConformance = new List { ProfileIdentifierType.software, ProfileIdentifierType.core, ProfileIdentifierType.simpleLicensing }, @@ -398,7 +394,7 @@ public GenerationResult GenerateJsonDocument(IInternalMetadataProvider internalM spdxDocument.AddSpdxId(); - var spdxRelationship = new Spdx30Relationship + var spdxRelationship = new Entities.Relationship { From = spdxDataLicense.SpdxId, RelationshipType = RelationshipType.HAS_DECLARED_LICENSE, @@ -485,7 +481,7 @@ private List GetSpdxRelationshipsFromSbomFile(Element spdxFileElement, var licenseConcludedElement = GenerateLicenseElement(fileInfo.LicenseConcluded); spdxRelationshipAndLicenseElementsToAddToSBOM.Add(licenseConcludedElement); - var spdxRelationshipLicenseConcludedElement = new Spdx30Relationship + var spdxRelationshipLicenseConcludedElement = new Entities.Relationship { From = spdxFileElement.SpdxId, RelationshipType = RelationshipType.HAS_CONCLUDED_LICENSE, @@ -503,7 +499,7 @@ private List GetSpdxRelationshipsFromSbomFile(Element spdxFileElement, toRelationships.Add(licenseDeclaredElement.SpdxId); } - var spdxRelationshipLicenseDeclaredElement = new Spdx30Relationship + var spdxRelationshipLicenseDeclaredElement = new Entities.Relationship { From = spdxFileElement.SpdxId, RelationshipType = RelationshipType.HAS_DECLARED_LICENSE, @@ -524,7 +520,7 @@ private List GetSpdxRelationshipsAndLicensesFromSbomPackage(SbomPackage var licenseConcludedElement = GenerateLicenseElement(packageInfo.LicenseInfo?.Concluded); spdxRelationshipAndLicenseElementsToAddToSBOM.Add(licenseConcludedElement); - var spdxRelationshipLicenseConcludedElement = new Spdx30Relationship + var spdxRelationshipLicenseConcludedElement = new Entities.Relationship { From = spdxPackage.SpdxId, RelationshipType = RelationshipType.HAS_CONCLUDED_LICENSE, @@ -538,7 +534,7 @@ private List GetSpdxRelationshipsAndLicensesFromSbomPackage(SbomPackage var licenseDeclaredElement = GenerateLicenseElement(packageInfo.LicenseInfo?.Declared); spdxRelationshipAndLicenseElementsToAddToSBOM.Add(licenseDeclaredElement); - var spdxRelationshipLicenseDeclaredElement = new Spdx30Relationship + var spdxRelationshipLicenseDeclaredElement = new Entities.Relationship { From = spdxPackage.SpdxId, RelationshipType = RelationshipType.HAS_DECLARED_LICENSE, diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/ContextsResult.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/ContextsResult.cs new file mode 100644 index 00000000..67f35068 --- /dev/null +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/ContextsResult.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.Sbom.JsonAsynchronousNodeKit; +using Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities; + +namespace Microsoft.Sbom.Parser; + +public record ContextsResult : ParserStateResult +{ + public ContextsResult(ParserStateResult result) + : base(result.FieldName, result.Result, result.ExplicitField, result.YieldReturn) + { + } + + public IEnumerable Contexts => ((IEnumerable)this.Result!).Select(r => r); +} diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/ElementsResult.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/ElementsResult.cs new file mode 100644 index 00000000..e9df538c --- /dev/null +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/ElementsResult.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.Sbom.JsonAsynchronousNodeKit; +using Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities; + +namespace Microsoft.Sbom.Parser; + +public record ElementsResult : ParserStateResult +{ + public ElementsResult(ParserStateResult result) + : base(result.FieldName, result.Result, result.ExplicitField, result.YieldReturn) + { + } + + public IEnumerable Elements => ((IEnumerable)this.Result!).Select(r => (Element)r); +} diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/ExternalMapsResult.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/ExternalMapsResult.cs new file mode 100644 index 00000000..bca7846e --- /dev/null +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/ExternalMapsResult.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.Sbom.JsonAsynchronousNodeKit; +using Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities; + +namespace Microsoft.Sbom.Parser; + +public record class ExternalMapsResult : ParserStateResult +{ + public ExternalMapsResult(ParserStateResult result) + : base(result.FieldName, result.Result, result.ExplicitField, result.YieldReturn) + { + } + + public IEnumerable References => ((IEnumerable)this.Result!).Select(r => (ExternalMap)r); +} diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/FilesResult.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/FilesResult.cs new file mode 100644 index 00000000..b3e9e0aa --- /dev/null +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/FilesResult.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.Sbom.JsonAsynchronousNodeKit; +using Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities; + +namespace Microsoft.Sbom.Parser; + +public record FilesResult : ParserStateResult +{ + public FilesResult(ParserStateResult result) + : base(result.FieldName, result.Result, result.ExplicitField, result.YieldReturn) + { + } + + public IEnumerable Files => ((IEnumerable)this.Result!).Select(r => (File)r); +} diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/PackagesResult.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/PackagesResult.cs new file mode 100644 index 00000000..9c3aa85b --- /dev/null +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/PackagesResult.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.Sbom.JsonAsynchronousNodeKit; +using Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities; + +namespace Microsoft.Sbom.Parser; + +public record PackagesResult : ParserStateResult +{ + public PackagesResult(ParserStateResult result) + : base(result.FieldName, result.Result, result.ExplicitField, result.YieldReturn) + { + } + + public IEnumerable Packages => ((IEnumerable)this.Result!).Select(r => (Package)r); +} diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/ParserResults.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/ParserResults.cs new file mode 100644 index 00000000..a932318a --- /dev/null +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/ParserResults.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Sbom.Parser; + +using System.Collections.Generic; +using Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities; + +public class ParserResults +{ + public FormatEnforcedSPDX3 FormatEnforcedSPDX3Result { get; set; } + + public int FilesCount = 0; + public int PackagesCount = 0; + public int ReferencesCount = 0; + public int RelationshipsCount = 0; +} diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/RelationshipsResult.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/RelationshipsResult.cs new file mode 100644 index 00000000..3520b5f3 --- /dev/null +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/RelationshipsResult.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.Sbom.JsonAsynchronousNodeKit; +using Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities; + +namespace Microsoft.Sbom.Parser; + +public record RelationshipsResult : ParserStateResult +{ + public RelationshipsResult(ParserStateResult result) + : base(result.FieldName, result.Result, result.ExplicitField, result.YieldReturn) + { + } + + public IEnumerable Relationships => ((IEnumerable)this.Result!).Select(r => (Relationship)r); +} diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs new file mode 100644 index 00000000..0e9a3a5b --- /dev/null +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs @@ -0,0 +1,156 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.Sbom.Contracts; +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.JsonAsynchronousNodeKit; +using Microsoft.Sbom.JsonAsynchronousNodeKit.Exceptions; +using Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities; +using SPDXConstants = Microsoft.Sbom.Parsers.Spdx30SbomParser.Constants; + +namespace Microsoft.Sbom.Parser; + +#nullable enable +/// +/// A parser for SPDX 3.0 SBOMs. +/// +/// +/// This class is not Thread-safe since the stream and JsonReaders assume a single forward-only reader. +/// Because of the use of recursion in , this class is also not suitable for parsing deeply nested json objects. +/// +public class SPDX30Parser : ISbomParser +{ + public const string ContextProperty = SPDXConstants.SPDXContextHeaderName; + public const string GraphProperty = SPDXConstants.SPDXGraphHeaderName; + public static readonly IReadOnlyCollection RequiredFields = new List + { + ContextProperty, + GraphProperty, + }; + + public string? RequiredComplianceStandard; + public IReadOnlyCollection? EntitiesToEnforceComplianceStandardsFor; + public SpdxMetadata Metadata = new SpdxMetadata(); + private readonly LargeJsonParser30 parser; + private readonly IList observedFieldNames = new List(); + private readonly bool requiredFieldsCheck = true; + private readonly JsonSerializerOptions jsonSerializerOptions; + private bool parsingComplete = false; + private readonly ManifestInfo spdxManifestInfo = new() + { + Name = SPDXConstants.SPDXName, + Version = SPDXConstants.SPDXVersion, + }; + + private readonly IReadOnlyCollection entitiesWithDifferentNTIARequirements = new List + { + "SpdxDocument", + "File", + "Package", + }; + + public SPDX30Parser( + Stream stream, + string? requiredComplianceStandard = null, + JsonSerializerOptions? jsonSerializerOptions = null, + int? bufferSize = null) + { + this.jsonSerializerOptions = jsonSerializerOptions ?? new JsonSerializerOptions + { + Converters = { new ElementSerializer() }, + }; + + var handlers = new Dictionary + { + { ContextProperty, new PropertyHandler(ParameterType.Array) }, + { GraphProperty, new PropertyHandler(ParameterType.Array) }, + }; + + switch (requiredComplianceStandard) + { + case "NTIA": + this.EntitiesToEnforceComplianceStandardsFor = this.entitiesWithDifferentNTIARequirements; + break; + } + + if (bufferSize is null) + { + this.parser = new LargeJsonParser30(stream, handlers, this.jsonSerializerOptions); + } + else + { + this.parser = new LargeJsonParser30(stream, handlers, this.jsonSerializerOptions, bufferSize.Value); + } + } + + /// + /// Return the result from the parser. + /// + /// null if parsing is complete, otherwise a representing the field which was visited. These results represent the root level fields of a json object. + /// If the field is an array the result will be an IEnumerable which you MUST fully enumerate before calling Next() again. + /// + /// + public ParserStateResult? Next() + { + ParserStateResult? result; + do + { + result = this.parser.Next(); + if (result is not null) + { + this.observedFieldNames.Add(result.FieldName); + if (result.Result is not null) + { + switch (result.FieldName) + { + case ContextProperty: + result = new ContextsResult(result); + break; + case GraphProperty: + result = new ElementsResult(result); + break; + default: + throw new InvalidDataException($"Explicit field {result.FieldName} is unhandled."); + } + + return result; + } + } + } + while (result is not null); + + if (this.requiredFieldsCheck) + { + foreach (var requiredField in RequiredFields) + { + if (!this.observedFieldNames.Contains(requiredField)) + { + throw new ParserException($"Required field {requiredField} was not found in the SPDX file"); + } + } + } + + this.parsingComplete = true; + + return null; + } + + public SpdxMetadata GetMetadata() + { + if (!this.parsingComplete) + { + throw new ParserException($"{nameof(this.GetMetadata)} can only be called after Parsing is complete to ensure that a whole object is returned."); + } + + return this.Metadata; + } + + public ManifestInfo[] RegisterManifest() => new ManifestInfo[] { this.spdxManifestInfo }; +} diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Utils/ElementSerializer.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Utils/ElementSerializer.cs index c47af346..145ea62e 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Utils/ElementSerializer.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Utils/ElementSerializer.cs @@ -6,14 +6,13 @@ using System.Text.Json; using System.Text.Json.Serialization; using System.Xml.Linq; +using Microsoft.Sbom.Contracts; using Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities; public class ElementSerializer : JsonConverter> { - public override List Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) - { + public override List Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => throw new NotImplementedException("Deserialization of Elements into specific subtypes is not implemented yet."); - } public override void Write(Utf8JsonWriter writer, List elements, JsonSerializerOptions options) { diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Utils/SPDXExtensions.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Utils/SPDXExtensions.cs index 0b8cfbfc..312d4a65 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Utils/SPDXExtensions.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Utils/SPDXExtensions.cs @@ -124,17 +124,12 @@ public static void AddSpdxId(this Element element) element.SpdxId = GenerateSpdxId(element, element.Name); } - public static void AddSpdxId(this NamespaceMap namespaceMap) - { - namespaceMap.SpdxId = GenerateSpdxId(namespaceMap, namespaceMap.Namespace); - } - public static void AddSpdxId(this CreationInfo creationInfo) { creationInfo.SpdxId = GenerateSpdxId(creationInfo, creationInfo.Id); } - public static void AddSpdxId(this Spdx30Relationship relationship) + public static void AddSpdxId(this Entities.Relationship relationship) { relationship.SpdxId = GenerateSpdxId(relationship, relationship.To + relationship.RelationshipType.ToString()); } diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Validator.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Validator.cs new file mode 100644 index 00000000..41310dc1 --- /dev/null +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Validator.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.IO; +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Parser; + +namespace Microsoft.Sbom.Parsers.Spdx30SbomParser; + +/// +/// Validates files in a folder against their checksums stored in an SPDX 2.2 SBOM. +/// +public class Validator : IManifestInterface +{ + public string Version { get; set; } + + private readonly ManifestInfo spdxManifestInfo = new() + { + Name = Constants.SPDXName, + Version = Constants.SPDXVersion + }; + + public ManifestData ParseManifest(string manifest) + => throw new NotImplementedException($"Currently we don't support parsing complete SPDX 2.2 SBOMs"); + + public ManifestInfo[] RegisterManifest() => new[] { spdxManifestInfo }; + + public ISbomParser CreateParser(Stream stream) => new SPDX30Parser(stream); +} diff --git a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/JsonStrings/SbomFullDocWithFilesStrings.cs b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/JsonStrings/SbomFullDocWithFilesStrings.cs new file mode 100644 index 00000000..4c017ef8 --- /dev/null +++ b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/JsonStrings/SbomFullDocWithFilesStrings.cs @@ -0,0 +1,459 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Sbom.Parser.JsonStrings; + +public static class SbomFullDocWithFilesStrings +{ + [System.Diagnostics.CodeAnalysis.SuppressMessage("Style", "JSON002:Probable JSON string detected", Justification = "Need to use JSON string")] + public const string SbomWithValidFileJsonString = + @" + { + ""@context"": [ + ""https://spdx.org/rdf/3.0.1/spdx-context.jsonld"" + ], + ""@graph"": [ + { + ""software_copyrightText"": ""sampleCopyright"", + ""creationInfo"": ""_:creationinfo"", + ""name"": ""./sample/path"", + ""spdxId"": ""SPDXRef-software_File-B4A9F99A3A03B9273AE34753D96564CB4F2B0FAD885BBD36B0DD619E9E8AC967"", + ""verifiedUsing"": [ + { + ""algorithm"": ""sha1"", + ""hashValue"": ""sha1value"", + ""creationInfo"": ""_:creationinfo"", + ""spdxId"": ""SPDXRef-PackageVerificationCode-B1565820A5CDAC40E0520D23F9D0B1497F240DDC51D72EAC6423D97D952D444F"", + ""type"": ""PackageVerificationCode"" + }, + { + ""algorithm"": ""sha256"", + ""hashValue"": ""sha256value"", + ""creationInfo"": ""_:creationinfo"", + ""spdxId"": ""SPDXRef-PackageVerificationCode-5D5B09F6DCB2D53A5FFFC60C4AC0D55FABDF556069D6631545F42AA6E3500F2E"", + ""type"": ""PackageVerificationCode"" + } + ], + ""type"": ""software_File"" + }, + { + ""creationInfo"": ""_:creationinfo"", + ""name"": ""Microsoft"", + ""spdxId"": ""SPDXRef-Organization-4B8D792FFFFCD3AF92D53A739B6DF98DF2B1F367C2745DDC0085B30F51EBBC81"", + ""type"": ""Organization"" + }, + { + ""creationInfo"": ""_:creationinfo"", + ""name"": ""Microsoft.SBOMTool-3.0.2-preview.0.41"", + ""spdxId"": ""SPDXRef-Tool-F3816A2B734CA08686741B17A8BC9020B8513FCE6A7BD33B1006102E2A1B55AA"", + ""type"": ""Tool"" + }, + { + ""@id"": ""_:creationinfo"", + ""created"": ""2023-05-11T00:24:54Z"", + ""createdBy"": [ + ""SPDXRef-Organization-4B8D792FFFFCD3AF92D53A739B6DF98DF2B1F367C2745DDC0085B30F51EBBC81"" + ], + ""createdUsing"": [ + ""SPDXRef-Tool-F3816A2B734CA08686741B17A8BC9020B8513FCE6A7BD33B1006102E2A1B55AA"" + ], + ""specVersion"": ""3.0"", + ""creationInfo"": ""_:creationinfo"", + ""spdxId"": ""SPDXRef-CreationInfo-0799B4D592549CF6159C30BA3E278BF063A6A241B8728C18E7AEC18BFC2CFF6F"", + ""type"": ""CreationInfo"" + }, + { + ""creationInfo"": ""_:creationinfo"", + ""namespaceMap"": { + ""sbom"": ""https://sbom.microsoft/1:EUb7DmXV0UyaZH1sttfV8A:n5KOcNVrWkGNryWx2sCN2A/13824:29192253/5qzhWC8k2k2wzStO28rMVQ"" + }, + ""rootElement"": [ + ""root-element-example"" + ], + ""dataLicense"": ""SPDXRef-AnyLicenseInfo-6E237C55B0583CB7BBA05562316C54B0A105ABA04775017E2253237B9A64613C"", + ""profileConformance"": [ + ""software"", + ""core"" + ], + ""name"": ""spdx-doc-name"", + ""spdxId"": ""SPDXRef-SpdxDocument-B93EED20C16A89A887B753958D42B794DD3C6570D3C2725B56B43477B38E05A1"", + ""type"": ""SpdxDocument"" + } + ] + } + "; + + [System.Diagnostics.CodeAnalysis.SuppressMessage("Style", "JSON002:Probable JSON string detected", Justification = "Need to use JSON string")] + public const string SbomFileWithMissingVerificationJsonString = + @" + { + ""@context"": [ + ""https://spdx.org/rdf/3.0.1/spdx-context.jsonld"" + ], + ""@graph"": [ + { + ""software_copyrightText"": ""sampleCopyright"", + ""creationInfo"": ""_:creationinfo"", + ""name"": ""./sample/path"", + ""spdxId"": ""SPDXRef-software_File-B4A9F99A3A03B9273AE34753D96564CB4F2B0FAD885BBD36B0DD619E9E8AC967"", + ""type"": ""software_File"" + }, + { + ""creationInfo"": ""_:creationinfo"", + ""name"": ""Microsoft"", + ""spdxId"": ""SPDXRef-Organization-4B8D792FFFFCD3AF92D53A739B6DF98DF2B1F367C2745DDC0085B30F51EBBC81"", + ""type"": ""Organization"" + }, + { + ""creationInfo"": ""_:creationinfo"", + ""name"": ""Microsoft.SBOMTool-3.0.2-preview.0.41"", + ""spdxId"": ""SPDXRef-Tool-F3816A2B734CA08686741B17A8BC9020B8513FCE6A7BD33B1006102E2A1B55AA"", + ""type"": ""Tool"" + }, + { + ""@id"": ""_:creationinfo"", + ""created"": ""2023-05-11T00:24:54Z"", + ""createdBy"": [ + ""SPDXRef-Organization-4B8D792FFFFCD3AF92D53A739B6DF98DF2B1F367C2745DDC0085B30F51EBBC81"" + ], + ""createdUsing"": [ + ""SPDXRef-Tool-F3816A2B734CA08686741B17A8BC9020B8513FCE6A7BD33B1006102E2A1B55AA"" + ], + ""specVersion"": ""3.0"", + ""creationInfo"": ""_:creationinfo"", + ""spdxId"": ""SPDXRef-CreationInfo-0799B4D592549CF6159C30BA3E278BF063A6A241B8728C18E7AEC18BFC2CFF6F"", + ""type"": ""CreationInfo"" + }, + { + ""creationInfo"": ""_:creationinfo"", + ""namespaceMap"": { + ""sbom"": ""https://sbom.microsoft/1:EUb7DmXV0UyaZH1sttfV8A:n5KOcNVrWkGNryWx2sCN2A/13824:29192253/5qzhWC8k2k2wzStO28rMVQ"" + }, + ""rootElement"": [ + ""root-element-example"" + ], + ""dataLicense"": ""SPDXRef-AnyLicenseInfo-6E237C55B0583CB7BBA05562316C54B0A105ABA04775017E2253237B9A64613C"", + ""profileConformance"": [ + ""software"", + ""core"" + ], + ""name"": ""spdx-doc-name"", + ""spdxId"": ""SPDXRef-SpdxDocument-B93EED20C16A89A887B753958D42B794DD3C6570D3C2725B56B43477B38E05A1"", + ""type"": ""SpdxDocument"" + } + ] + } + "; + + [System.Diagnostics.CodeAnalysis.SuppressMessage("Style", "JSON002:Probable JSON string detected", Justification = "Need to use JSON string")] + public const string SbomFileWithMissingSHA256JsonString = + @" + { + ""@context"": [ + ""https://spdx.org/rdf/3.0.1/spdx-context.jsonld"" + ], + ""@graph"": [ + { + ""software_copyrightText"": ""sampleCopyright"", + ""creationInfo"": ""_:creationinfo"", + ""verifiedUsing"": [ + { + ""algorithm"": ""sha1"", + ""hashValue"": ""sha1value"", + ""creationInfo"": ""_:creationinfo"", + ""spdxId"": ""SPDXRef-PackageVerificationCode-B1565820A5CDAC40E0520D23F9D0B1497F240DDC51D72EAC6423D97D952D444F"", + ""type"": ""PackageVerificationCode"" + } + ], + ""name"": ""./sample/path"", + ""spdxId"": ""SPDXRef-software_File-B4A9F99A3A03B9273AE34753D96564CB4F2B0FAD885BBD36B0DD619E9E8AC967"", + ""type"": ""software_File"" + }, + { + ""creationInfo"": ""_:creationinfo"", + ""name"": ""Microsoft"", + ""spdxId"": ""SPDXRef-Organization-4B8D792FFFFCD3AF92D53A739B6DF98DF2B1F367C2745DDC0085B30F51EBBC81"", + ""type"": ""Organization"" + }, + { + ""creationInfo"": ""_:creationinfo"", + ""name"": ""Microsoft.SBOMTool-3.0.2-preview.0.41"", + ""spdxId"": ""SPDXRef-Tool-F3816A2B734CA08686741B17A8BC9020B8513FCE6A7BD33B1006102E2A1B55AA"", + ""type"": ""Tool"" + }, + { + ""@id"": ""_:creationinfo"", + ""created"": ""2023-05-11T00:24:54Z"", + ""createdBy"": [ + ""SPDXRef-Organization-4B8D792FFFFCD3AF92D53A739B6DF98DF2B1F367C2745DDC0085B30F51EBBC81"" + ], + ""createdUsing"": [ + ""SPDXRef-Tool-F3816A2B734CA08686741B17A8BC9020B8513FCE6A7BD33B1006102E2A1B55AA"" + ], + ""specVersion"": ""3.0"", + ""creationInfo"": ""_:creationinfo"", + ""spdxId"": ""SPDXRef-CreationInfo-0799B4D592549CF6159C30BA3E278BF063A6A241B8728C18E7AEC18BFC2CFF6F"", + ""type"": ""CreationInfo"" + }, + { + ""creationInfo"": ""_:creationinfo"", + ""namespaceMap"": { + ""sbom"": ""https://sbom.microsoft/1:EUb7DmXV0UyaZH1sttfV8A:n5KOcNVrWkGNryWx2sCN2A/13824:29192253/5qzhWC8k2k2wzStO28rMVQ"" + }, + ""rootElement"": [ + ""root-element-example"" + ], + ""dataLicense"": ""SPDXRef-AnyLicenseInfo-6E237C55B0583CB7BBA05562316C54B0A105ABA04775017E2253237B9A64613C"", + ""profileConformance"": [ + ""software"", + ""core"" + ], + ""name"": ""spdx-doc-name"", + ""spdxId"": ""SPDXRef-SpdxDocument-B93EED20C16A89A887B753958D42B794DD3C6570D3C2725B56B43477B38E05A1"", + ""type"": ""SpdxDocument"" + } + ] + } + "; + + [System.Diagnostics.CodeAnalysis.SuppressMessage("Style", "JSON002:Probable JSON string detected", Justification = "Need to use JSON string")] + public const string SbomFileWithMissingNameJsonString = + @" + { + ""@context"": [ + ""https://spdx.org/rdf/3.0.1/spdx-context.jsonld"" + ], + ""@graph"": [ + { + ""software_copyrightText"": ""sampleCopyright"", + ""creationInfo"": ""_:creationinfo"", + ""spdxId"": ""SPDXRef-software_File-B4A9F99A3A03B9273AE34753D96564CB4F2B0FAD885BBD36B0DD619E9E8AC967"", + ""verifiedUsing"": [ + { + ""algorithm"": ""sha1"", + ""hashValue"": ""sha1value"", + ""creationInfo"": ""_:creationinfo"", + ""spdxId"": ""SPDXRef-PackageVerificationCode-B1565820A5CDAC40E0520D23F9D0B1497F240DDC51D72EAC6423D97D952D444F"", + ""type"": ""PackageVerificationCode"" + }, + { + ""algorithm"": ""sha256"", + ""hashValue"": ""sha256value"", + ""creationInfo"": ""_:creationinfo"", + ""spdxId"": ""SPDXRef-PackageVerificationCode-5D5B09F6DCB2D53A5FFFC60C4AC0D55FABDF556069D6631545F42AA6E3500F2E"", + ""type"": ""PackageVerificationCode"" + } + ], + ""type"": ""software_File"" + }, + { + ""creationInfo"": ""_:creationinfo"", + ""name"": ""Microsoft"", + ""spdxId"": ""SPDXRef-Organization-4B8D792FFFFCD3AF92D53A739B6DF98DF2B1F367C2745DDC0085B30F51EBBC81"", + ""type"": ""Organization"" + }, + { + ""creationInfo"": ""_:creationinfo"", + ""name"": ""Microsoft.SBOMTool-3.0.2-preview.0.41"", + ""spdxId"": ""SPDXRef-Tool-F3816A2B734CA08686741B17A8BC9020B8513FCE6A7BD33B1006102E2A1B55AA"", + ""type"": ""Tool"" + }, + { + ""@id"": ""_:creationinfo"", + ""created"": ""2023-05-11T00:24:54Z"", + ""createdBy"": [ + ""SPDXRef-Organization-4B8D792FFFFCD3AF92D53A739B6DF98DF2B1F367C2745DDC0085B30F51EBBC81"" + ], + ""createdUsing"": [ + ""SPDXRef-Tool-F3816A2B734CA08686741B17A8BC9020B8513FCE6A7BD33B1006102E2A1B55AA"" + ], + ""specVersion"": ""3.0"", + ""creationInfo"": ""_:creationinfo"", + ""spdxId"": ""SPDXRef-CreationInfo-0799B4D592549CF6159C30BA3E278BF063A6A241B8728C18E7AEC18BFC2CFF6F"", + ""type"": ""CreationInfo"" + }, + { + ""creationInfo"": ""_:creationinfo"", + ""namespaceMap"": { + ""sbom"": ""https://sbom.microsoft/1:EUb7DmXV0UyaZH1sttfV8A:n5KOcNVrWkGNryWx2sCN2A/13824:29192253/5qzhWC8k2k2wzStO28rMVQ"" + }, + ""rootElement"": [ + ""root-element-example"" + ], + ""dataLicense"": ""SPDXRef-AnyLicenseInfo-6E237C55B0583CB7BBA05562316C54B0A105ABA04775017E2253237B9A64613C"", + ""profileConformance"": [ + ""software"", + ""core"" + ], + ""name"": ""spdx-doc-name"", + ""spdxId"": ""SPDXRef-SpdxDocument-B93EED20C16A89A887B753958D42B794DD3C6570D3C2725B56B43477B38E05A1"", + ""type"": ""SpdxDocument"" + } + ] + } + "; + + [System.Diagnostics.CodeAnalysis.SuppressMessage("Style", "JSON002:Probable JSON string detected", Justification = "Need to use JSON string")] + public const string SbomFileWithMissingSpdxIdJsonString = + @" + { + ""@context"": [ + ""https://spdx.org/rdf/3.0.1/spdx-context.jsonld"" + ], + ""@graph"": [ + { + ""software_copyrightText"": ""sampleCopyright"", + ""creationInfo"": ""_:creationinfo"", + ""name"": ""./sample/path"", + ""verifiedUsing"": [ + { + ""algorithm"": ""sha1"", + ""hashValue"": ""sha1value"", + ""creationInfo"": ""_:creationinfo"", + ""spdxId"": ""SPDXRef-PackageVerificationCode-B1565820A5CDAC40E0520D23F9D0B1497F240DDC51D72EAC6423D97D952D444F"", + ""type"": ""PackageVerificationCode"" + }, + { + ""algorithm"": ""sha256"", + ""hashValue"": ""sha256value"", + ""creationInfo"": ""_:creationinfo"", + ""spdxId"": ""SPDXRef-PackageVerificationCode-5D5B09F6DCB2D53A5FFFC60C4AC0D55FABDF556069D6631545F42AA6E3500F2E"", + ""type"": ""PackageVerificationCode"" + } + ], + ""type"": ""software_File"" + }, + { + ""creationInfo"": ""_:creationinfo"", + ""name"": ""Microsoft"", + ""spdxId"": ""SPDXRef-Organization-4B8D792FFFFCD3AF92D53A739B6DF98DF2B1F367C2745DDC0085B30F51EBBC81"", + ""type"": ""Organization"" + }, + { + ""creationInfo"": ""_:creationinfo"", + ""name"": ""Microsoft.SBOMTool-3.0.2-preview.0.41"", + ""spdxId"": ""SPDXRef-Tool-F3816A2B734CA08686741B17A8BC9020B8513FCE6A7BD33B1006102E2A1B55AA"", + ""type"": ""Tool"" + }, + { + ""@id"": ""_:creationinfo"", + ""created"": ""2023-05-11T00:24:54Z"", + ""createdBy"": [ + ""SPDXRef-Organization-4B8D792FFFFCD3AF92D53A739B6DF98DF2B1F367C2745DDC0085B30F51EBBC81"" + ], + ""createdUsing"": [ + ""SPDXRef-Tool-F3816A2B734CA08686741B17A8BC9020B8513FCE6A7BD33B1006102E2A1B55AA"" + ], + ""specVersion"": ""3.0"", + ""creationInfo"": ""_:creationinfo"", + ""spdxId"": ""SPDXRef-CreationInfo-0799B4D592549CF6159C30BA3E278BF063A6A241B8728C18E7AEC18BFC2CFF6F"", + ""type"": ""CreationInfo"" + }, + { + ""creationInfo"": ""_:creationinfo"", + ""namespaceMap"": { + ""sbom"": ""https://sbom.microsoft/1:EUb7DmXV0UyaZH1sttfV8A:n5KOcNVrWkGNryWx2sCN2A/13824:29192253/5qzhWC8k2k2wzStO28rMVQ"" + }, + ""rootElement"": [ + ""root-element-example"" + ], + ""dataLicense"": ""SPDXRef-AnyLicenseInfo-6E237C55B0583CB7BBA05562316C54B0A105ABA04775017E2253237B9A64613C"", + ""profileConformance"": [ + ""software"", + ""core"" + ], + ""name"": ""spdx-doc-name"", + ""spdxId"": ""SPDXRef-SpdxDocument-B93EED20C16A89A887B753958D42B794DD3C6570D3C2725B56B43477B38E05A1"", + ""type"": ""SpdxDocument"" + } + ] + } + "; + + [System.Diagnostics.CodeAnalysis.SuppressMessage("Style", "JSON002:Probable JSON string detected", Justification = "Need to use JSON string")] + public const string SbomFileWithAdditionalPropertiesJsonString = + @" + { + ""@context"": [ + ""https://spdx.org/rdf/3.0.1/spdx-context.jsonld"" + ], + ""@graph"": [ + { + ""software_copyrightText"": ""sampleCopyright"", + ""creationInfo"": ""_:creationinfo"", + ""name"": ""./sample/path"", + ""spdxId"": ""SPDXRef-software_File-B4A9F99A3A03B9273AE34753D96564CB4F2B0FAD885BBD36B0DD619E9E8AC967"", + ""verifiedUsing"": [ + { + ""algorithm"": ""sha1"", + ""hashValue"": ""sha1value"", + ""creationInfo"": ""_:creationinfo"", + ""spdxId"": ""SPDXRef-PackageVerificationCode-B1565820A5CDAC40E0520D23F9D0B1497F240DDC51D72EAC6423D97D952D444F"", + ""type"": ""PackageVerificationCode"" + }, + { + ""algorithm"": ""sha256"", + ""hashValue"": ""sha256value"", + ""creationInfo"": ""_:creationinfo"", + ""spdxId"": ""SPDXRef-PackageVerificationCode-5D5B09F6DCB2D53A5FFFC60C4AC0D55FABDF556069D6631545F42AA6E3500F2E"", + ""type"": ""PackageVerificationCode"" + } + ], + ""type"": ""software_File"", + ""additionalProperty"": ""additionalValue"", + ""additionalPropertyArray"": [""additionalValue1"", ""additionalValue2""], + ""additionalPropertyObject"": {""additionalPropertyObjectKey"": ""additionalPropertyObjectValue""}, + ""additionalPropertyWithArrayChildProperty"": [ + {""childAddtionalProperty"": ""Additional property value""} + ], + ""additionalPropertyWithObjectChildProperty"": { + ""childAddtionalProperty"": ""Additional property value"" + } + }, + { + ""creationInfo"": ""_:creationinfo"", + ""name"": ""Microsoft"", + ""spdxId"": ""SPDXRef-Organization-4B8D792FFFFCD3AF92D53A739B6DF98DF2B1F367C2745DDC0085B30F51EBBC81"", + ""type"": ""Organization"" + }, + { + ""creationInfo"": ""_:creationinfo"", + ""name"": ""Microsoft.SBOMTool-3.0.2-preview.0.41"", + ""spdxId"": ""SPDXRef-Tool-F3816A2B734CA08686741B17A8BC9020B8513FCE6A7BD33B1006102E2A1B55AA"", + ""type"": ""Tool"" + }, + { + ""@id"": ""_:creationinfo"", + ""created"": ""2023-05-11T00:24:54Z"", + ""createdBy"": [ + ""SPDXRef-Organization-4B8D792FFFFCD3AF92D53A739B6DF98DF2B1F367C2745DDC0085B30F51EBBC81"" + ], + ""createdUsing"": [ + ""SPDXRef-Tool-F3816A2B734CA08686741B17A8BC9020B8513FCE6A7BD33B1006102E2A1B55AA"" + ], + ""specVersion"": ""3.0"", + ""creationInfo"": ""_:creationinfo"", + ""spdxId"": ""SPDXRef-CreationInfo-0799B4D592549CF6159C30BA3E278BF063A6A241B8728C18E7AEC18BFC2CFF6F"", + ""type"": ""CreationInfo"" + }, + { + ""creationInfo"": ""_:creationinfo"", + ""namespaceMap"": { + ""sbom"": ""https://sbom.microsoft/1:EUb7DmXV0UyaZH1sttfV8A:n5KOcNVrWkGNryWx2sCN2A/13824:29192253/5qzhWC8k2k2wzStO28rMVQ"" + }, + ""rootElement"": [ + ""root-element-example"" + ], + ""dataLicense"": ""SPDXRef-AnyLicenseInfo-6E237C55B0583CB7BBA05562316C54B0A105ABA04775017E2253237B9A64613C"", + ""profileConformance"": [ + ""software"", + ""core"" + ], + ""name"": ""spdx-doc-name"", + ""spdxId"": ""SPDXRef-SpdxDocument-B93EED20C16A89A887B753958D42B794DD3C6570D3C2725B56B43477B38E05A1"", + ""type"": ""SpdxDocument"" + } + ] + } + "; +} diff --git a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/JsonStrings/SbomFullDocWithMetadataJsonStrings.cs b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/JsonStrings/SbomFullDocWithMetadataJsonStrings.cs new file mode 100644 index 00000000..fdc8807a --- /dev/null +++ b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/JsonStrings/SbomFullDocWithMetadataJsonStrings.cs @@ -0,0 +1,197 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Sbom.Parser.JsonStrings; + +public static class SbomFullDocWithMetadataJsonStrings +{ + [System.Diagnostics.CodeAnalysis.SuppressMessage("Style", "JSON002:Probable JSON string detected", Justification = "Need to use JSON string")] + public const string SbomWithValidMetadataJsonString = + @" + { + ""@context"": [ + ""https://spdx.org/rdf/3.0.1/spdx-context.jsonld"" + ], + ""@graph"": [ + { + ""creationInfo"": ""_:creationinfo"", + ""name"": ""Microsoft"", + ""spdxId"": ""SPDXRef-Organization-4B8D792FFFFCD3AF92D53A739B6DF98DF2B1F367C2745DDC0085B30F51EBBC81"", + ""type"": ""Organization"" + }, + { + ""creationInfo"": ""_:creationinfo"", + ""name"": ""Microsoft.SBOMTool-3.0.2-preview.0.41"", + ""spdxId"": ""SPDXRef-Tool-F3816A2B734CA08686741B17A8BC9020B8513FCE6A7BD33B1006102E2A1B55AA"", + ""type"": ""Tool"" + }, + { + ""@id"": ""_:creationinfo"", + ""created"": ""2023-05-11T00:24:54Z"", + ""createdBy"": [ + ""SPDXRef-Organization-4B8D792FFFFCD3AF92D53A739B6DF98DF2B1F367C2745DDC0085B30F51EBBC81"" + ], + ""createdUsing"": [ + ""SPDXRef-Tool-F3816A2B734CA08686741B17A8BC9020B8513FCE6A7BD33B1006102E2A1B55AA"" + ], + ""specVersion"": ""3.0"", + ""creationInfo"": ""_:creationinfo"", + ""spdxId"": ""SPDXRef-CreationInfo-0799B4D592549CF6159C30BA3E278BF063A6A241B8728C18E7AEC18BFC2CFF6F"", + ""type"": ""CreationInfo"" + }, + { + ""creationInfo"": ""_:creationinfo"", + ""name"": ""CC0-1.0"", + ""spdxId"": ""SPDXRef-AnyLicenseInfo-6E237C55B0583CB7BBA05562316C54B0A105ABA04775017E2253237B9A64613C"", + ""type"": ""AnyLicenseInfo"" + }, + { + ""creationInfo"": ""_:creationinfo"", + ""namespaceMap"": { + ""sbom"": ""https://sbom.microsoft/1:EUb7DmXV0UyaZH1sttfV8A:n5KOcNVrWkGNryWx2sCN2A/13824:29192253/5qzhWC8k2k2wzStO28rMVQ"" + }, + ""rootElement"": [ + ""root-element-example"" + ], + ""dataLicense"": ""SPDXRef-AnyLicenseInfo-6E237C55B0583CB7BBA05562316C54B0A105ABA04775017E2253237B9A64613C"", + ""profileConformance"": [ + ""software"", + ""core"" + ], + ""name"": ""spdx-doc-name"", + ""spdxId"": ""SPDXRef-SpdxDocument-B93EED20C16A89A887B753958D42B794DD3C6570D3C2725B56B43477B38E05A1"", + ""type"": ""SpdxDocument"" + } + ] + } + "; + + [System.Diagnostics.CodeAnalysis.SuppressMessage("Style", "JSON002:Probable JSON string detected", Justification = "Need to use JSON string")] + public const string SbomWithSpdxDocumentMissingNameJsonString = + @" + { + ""@context"": [ + ""https://spdx.org/rdf/3.0.1/spdx-context.jsonld"" + ], + ""@graph"": [ + { + ""creationInfo"": ""_:creationinfo"", + ""name"": ""Microsoft"", + ""spdxId"": ""SPDXRef-Organization-4B8D792FFFFCD3AF92D53A739B6DF98DF2B1F367C2745DDC0085B30F51EBBC81"", + ""type"": ""Organization"" + }, + { + ""creationInfo"": ""_:creationinfo"", + ""name"": ""Microsoft.SBOMTool-3.0.2-preview.0.41"", + ""spdxId"": ""SPDXRef-Tool-F3816A2B734CA08686741B17A8BC9020B8513FCE6A7BD33B1006102E2A1B55AA"", + ""type"": ""Tool"" + }, + { + ""@id"": ""_:creationinfo"", + ""created"": ""2023-05-11T00:24:54Z"", + ""createdBy"": [ + ""SPDXRef-Organization-4B8D792FFFFCD3AF92D53A739B6DF98DF2B1F367C2745DDC0085B30F51EBBC81"" + ], + ""createdUsing"": [ + ""SPDXRef-Tool-F3816A2B734CA08686741B17A8BC9020B8513FCE6A7BD33B1006102E2A1B55AA"" + ], + ""specVersion"": ""3.0"", + ""creationInfo"": ""_:creationinfo"", + ""spdxId"": ""SPDXRef-CreationInfo-0799B4D592549CF6159C30BA3E278BF063A6A241B8728C18E7AEC18BFC2CFF6F"", + ""type"": ""CreationInfo"" + }, + { + ""creationInfo"": ""_:creationinfo"", + ""name"": ""CC0-1.0"", + ""spdxId"": ""SPDXRef-AnyLicenseInfo-6E237C55B0583CB7BBA05562316C54B0A105ABA04775017E2253237B9A64613C"", + ""type"": ""AnyLicenseInfo"" + }, + { + ""creationInfo"": ""_:creationinfo"", + ""namespaceMap"": { + ""sbom"": ""https://sbom.microsoft/1:EUb7DmXV0UyaZH1sttfV8A:n5KOcNVrWkGNryWx2sCN2A/13824:29192253/5qzhWC8k2k2wzStO28rMVQ"" + }, + ""rootElement"": [ + ""root-element-example"" + ], + ""dataLicense"": ""SPDXRef-AnyLicenseInfo-6E237C55B0583CB7BBA05562316C54B0A105ABA04775017E2253237B9A64613C"", + ""profileConformance"": [ + ""software"", + ""core"" + ], + ""spdxId"": ""SPDXRef-SpdxDocument-B93EED20C16A89A887B753958D42B794DD3C6570D3C2725B56B43477B38E05A1"", + ""type"": ""SpdxDocument"" + } + ] + } + "; + + [System.Diagnostics.CodeAnalysis.SuppressMessage("Style", "JSON002:Probable JSON string detected", Justification = "Need to use JSON string")] + public const string SbomWithInvalidContextJsonString = + @" + { + ""@context"": [ + ""https://spdx.org/rdf/3.0.1/spdx-context.jsonld"", + ""https://spdx.org/rdf/3.0.1/spdx-context.jsonld"" + ], + ""@graph"": [ + { + ""creationInfo"": ""_:creationinfo"", + ""name"": ""the-package-namethe-package-version"", + ""spdxId"": ""SPDXRef-SpdxDocument-B93EED20C16A89A887B753958D42B794DD3C6570D3C2725B56B43477B38E05A1"", + ""type"": ""SpdxDocument"" + } + ] + } + "; + + public const string MalformedJsonEmptyObject = + @" + { + ""@context"": [ + ""https://spdx.org/rdf/3.0.1/spdx-context.jsonld"" + ], + ""@graph"": + { + }"; + + [System.Diagnostics.CodeAnalysis.SuppressMessage("Style", "JSON002:Probable JSON string detected", Justification = "Need to use JSON string")] + public const string MalformedJsonEmptyArray = + @" + { + ""@context"": [ + ""https://spdx.org/rdf/3.0.1/spdx-context.jsonld"" + ], + ""@graph"": + [ + { + ""creationInfo"": ""_:creationinfo"", + ""namespaceMap"": { + ""sbom"": ""https://sbom.microsoft/1:EUb7DmXV0UyaZH1sttfV8A:n5KOcNVrWkGNryWx2sCN2A/13824:29192253/5qzhWC8k2k2wzStO28rMVQ"" + }, + ""rootElement"": [ + ""root-element-example"" + ], + ""dataLicense"": ""SPDXRef-AnyLicenseInfo-6E237C55B0583CB7BBA05562316C54B0A105ABA04775017E2253237B9A64613C"", + ""profileConformance"": [ + ""software"", + ""core"" + ], + ""name"": ""spdx-doc-name"", + ""spdxId"": ""SPDXRef-SpdxDocument-B93EED20C16A89A887B753958D42B794DD3C6570D3C2725B56B43477B38E05A1"", + ""type"": ""SpdxDocument"" + } + }"; + + [System.Diagnostics.CodeAnalysis.SuppressMessage("Style", "JSON002:Probable JSON string detected", Justification = "Need to use JSON string")] + + public const string JsonEmptyArray = + @" + { + ""@context"": [ + ""https://spdx.org/rdf/3.0.1/spdx-context.jsonld"" + ], + ""@graph"": [ + ] + }"; +} diff --git a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomFileParserTests.cs b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomFileParserTests.cs new file mode 100644 index 00000000..c13a1fde --- /dev/null +++ b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomFileParserTests.cs @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.IO; +using System.Linq; +using System.Text; +using Microsoft.Sbom.JsonAsynchronousNodeKit.Exceptions; +using Microsoft.Sbom.Parser.JsonStrings; +using Microsoft.Sbom.Parsers.Spdx30SbomParser; +using Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities; +using Microsoft.Sbom.Utils; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using File = Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities.File; + +namespace Microsoft.Sbom.Parser; + +[TestClass] +public class SbomFileParserTests : SbomParserTestsBase +{ + [DataTestMethod] + [DataRow(SbomFullDocWithFilesStrings.SbomFileWithMissingNameJsonString)] + [DataRow(SbomFullDocWithFilesStrings.SbomFileWithMissingSpdxIdJsonString)] + [TestMethod] + public void MissingPropertiesTest_Throws(string json) + { + var bytes = Encoding.UTF8.GetBytes(json); + using var stream = new MemoryStream(bytes); + var parser = new SPDX30Parser(stream); + _ = Assert.ThrowsException(() => this.Parse(parser)); + } + + [TestMethod] + public void MissingPropertiesTest_NTIA_NoVerificationCode_Throws() + { + var bytes = Encoding.UTF8.GetBytes(SbomFullDocWithFilesStrings.SbomFileWithMissingVerificationJsonString); + using var stream = new MemoryStream(bytes); + var parser = new SPDX30Parser(stream, requiredComplianceStandard: "NTIA"); + Assert.ThrowsException(() => this.Parse(parser)); + } + + [TestMethod] + public void MissingPropertiesTest_NTIA_VerificationCodeWithNoSHA256_Throws() + { + var bytes = Encoding.UTF8.GetBytes(SbomFullDocWithFilesStrings.SbomFileWithMissingSHA256JsonString); + using var stream = new MemoryStream(bytes); + var parser = new SPDX30Parser(stream, requiredComplianceStandard: "NTIA"); + Assert.ThrowsException(() => this.Parse(parser)); + } + + [TestMethod] + public void IgnoresAdditionalPropertiesTest() + { + var bytes = Encoding.UTF8.GetBytes(SbomFullDocWithFilesStrings.SbomFileWithAdditionalPropertiesJsonString); + using var stream = new MemoryStream(bytes); + var parser = new SPDX30Parser(stream); + var result = this.Parse(parser); + Assert.AreEqual(1, result.FilesCount); + } +} diff --git a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomMetadataParserTests.cs b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomMetadataParserTests.cs new file mode 100644 index 00000000..14e08fd7 --- /dev/null +++ b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomMetadataParserTests.cs @@ -0,0 +1,162 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using Microsoft.Sbom.JsonAsynchronousNodeKit.Exceptions; +using Microsoft.Sbom.Parser.JsonStrings; +using Microsoft.Sbom.Parsers.Spdx30SbomParser; +using Microsoft.Sbom.Utils; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Microsoft.Sbom.Parser; + +[TestClass] +public class SbomMetadataParserTests : SbomParserTestsBase +{ + [TestMethod] + public void MetadataPopulates() + { + var bytes = Encoding.UTF8.GetBytes(SbomFullDocWithMetadataJsonStrings.SbomWithValidMetadataJsonString); + using var stream = new MemoryStream(bytes); + + var parser = new SPDX30Parser(stream); + + var results = this.Parse(parser); + Assert.IsTrue(results.FormatEnforcedSPDX3Result.Graph.Count() == 5); + + var metadata = parser.GetMetadata(); + Assert.IsNotNull(metadata); + Assert.IsTrue(metadata.DocumentNamespace != null); + Assert.IsTrue(metadata.Name == "spdx-doc-name"); + Assert.IsTrue(metadata.SpdxId == "SPDXRef-SpdxDocument-B93EED20C16A89A887B753958D42B794DD3C6570D3C2725B56B43477B38E05A1"); + Assert.IsTrue(metadata.DocumentDescribes.First() == "root-element-example"); + Assert.IsTrue(metadata.DataLicense == "CC0-1.0"); + Assert.IsTrue(metadata.CreationInfo.Creators.Count() == 2); + Assert.IsTrue(metadata.CreationInfo.Created != DateTime.MinValue); + Assert.IsTrue(metadata.SpdxVersion == "3.0"); + } + + [TestMethod] + public void MetadataEmpty() + { + var bytes = Encoding.UTF8.GetBytes(SbomFullDocWithMetadataJsonStrings.JsonEmptyArray); + using var stream = new MemoryStream(bytes); + + var parser = new SPDX30Parser(stream); + + var results = this.Parse(parser); + + var metadata = parser.GetMetadata(); + Assert.IsNotNull(metadata); + Assert.IsTrue(metadata.DocumentNamespace == null); + Assert.IsTrue(metadata.Name == null); + Assert.IsTrue(metadata.SpdxId == null); + Assert.IsTrue(metadata.DocumentDescribes == null); + Assert.IsTrue(metadata.DataLicense == null); + Assert.IsTrue(metadata.CreationInfo == null); + Assert.IsTrue(metadata.SpdxVersion == null); + } + + [TestMethod] + public void DocCreation_NoName_NTIA_Throws() + { + var bytes = Encoding.UTF8.GetBytes(SbomFullDocWithMetadataJsonStrings.SbomWithSpdxDocumentMissingNameJsonString); + using var stream = new MemoryStream(bytes); + + var parser = new SPDX30Parser(stream, requiredComplianceStandard: "NTIA"); + Assert.ThrowsException(() => this.Parse(parser)); + } + + [TestMethod] + public void DocCreation_NoName_Passes() + { + var bytes = Encoding.UTF8.GetBytes(SbomFullDocWithMetadataJsonStrings.SbomWithSpdxDocumentMissingNameJsonString); + using var stream = new MemoryStream(bytes); + + var parser = new SPDX30Parser(stream); + var results = this.Parse(parser); + Assert.IsTrue(results.FormatEnforcedSPDX3Result.Graph.Count() == 5); + + var metadata = parser.GetMetadata(); + Assert.IsNotNull(metadata); + } + + [TestMethod] + public void InvalidContextThrows() + { + var bytes = Encoding.UTF8.GetBytes(SbomFullDocWithMetadataJsonStrings.SbomWithInvalidContextJsonString); + using var stream = new MemoryStream(bytes); + + var parser = new SPDX30Parser(stream); + Assert.ThrowsException(() => this.Parse(parser)); + } + + [TestMethod] + public void NullStreamThrows() + { + _ = Assert.ThrowsException(() => new SPDXParser(null)); + } + + [TestMethod] + public void StreamClosedTestReturnsNull() + { + var bytes = Encoding.UTF8.GetBytes(SbomFullDocWithMetadataJsonStrings.SbomWithValidMetadataJsonString); + using var stream = new MemoryStream(bytes); + + var parser = new SPDX30Parser(stream); + + Assert.ThrowsException(() => this.Parse(parser, stream, close: true)); + } + + [TestMethod] + public void StreamEmptyTestReturnsNull() + { + using var stream = new MemoryStream(); + stream.Read(new byte[Constants.ReadBufferSize]); + var buffer = new byte[Constants.ReadBufferSize]; + + Assert.ThrowsException(() => new SPDXParser(stream)); + } + + [DataTestMethod] + [DataRow(SbomFullDocWithMetadataJsonStrings.MalformedJsonEmptyObject)] + [DataRow(SbomFullDocWithMetadataJsonStrings.MalformedJsonEmptyArray)] + [TestMethod] + public void MalformedJsonTest_Throws(string json) + { + var bytes = Encoding.UTF8.GetBytes(json); + using var stream = new MemoryStream(bytes); + + var parser = new SPDX30Parser(stream); + Assert.ThrowsException(() => this.Parse(parser)); + } + + [TestMethod] + public void EmptyArray_ValidJson() + { + var bytes = Encoding.UTF8.GetBytes(SbomFullDocWithMetadataJsonStrings.JsonEmptyArray); + using var stream = new MemoryStream(bytes); + + var parser = new SPDX30Parser(stream); + + var result = this.Parse(parser); + + Assert.AreEqual(0, result.FilesCount); + } + + [TestMethod] + [ExpectedException(typeof(ArgumentException))] + public void NullOrEmptyBuffer_Throws() + { + var bytes = Encoding.UTF8.GetBytes(SbomFullDocWithMetadataJsonStrings.MalformedJsonEmptyObject); + using var stream = new MemoryStream(bytes); + + var parser = new SPDX30Parser(stream, bufferSize: 0); + this.Parse(parser); + Assert.ThrowsException(() => this.Parse(parser)); + } +} diff --git a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomParserTestsBase.cs b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomParserTestsBase.cs new file mode 100644 index 00000000..67a061ec --- /dev/null +++ b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomParserTestsBase.cs @@ -0,0 +1,311 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Linq.Expressions; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Sbom.Api.Entities.Output; +using Microsoft.Sbom.Contracts; +using Microsoft.Sbom.JsonAsynchronousNodeKit; +using Microsoft.Sbom.JsonAsynchronousNodeKit.Exceptions; +using Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities; +using Newtonsoft.Json.Linq; + +namespace Microsoft.Sbom.Parser; + +#nullable enable + +public abstract class SbomParserTestsBase +{ + public ParserResults Parse(SPDX30Parser parser, Stream? stream = null, bool close = false) + { + var results = new ParserResults(); + + ParserStateResult? result = null; + do + { + result = parser.Next(); + + if (close) + { + if (stream is not null) + { + stream.Close(); + } + else + { + throw new NotImplementedException("Can't close a stream without the stream."); + } + } + + if (result is not null && result.Result is not null) + { + var jsonElements = result.Result as IEnumerable; + var jsonList = jsonElements?.ToList(); + results.FormatEnforcedSPDX3Result ??= new FormatEnforcedSPDX3(); + switch (result.FieldName) + { + case SPDX30Parser.ContextProperty: + if (jsonList == null || !jsonList.Any() || jsonList.Count > 1) + { + throw new ParserException($"The context property is either empty or has more than one string."); + } + else + { + results.FormatEnforcedSPDX3Result.Context = (string)jsonList.First(); + } + + break; + case SPDX30Parser.GraphProperty: + results.FormatEnforcedSPDX3Result.Graph = ConvertToElements(jsonList, ref results, parser.RequiredComplianceStandard, parser.EntitiesToEnforceComplianceStandardsFor); + parser.Metadata = this.SetMetadata(results); + break; + default: + Console.WriteLine($"Unrecognized FieldName: {result.FieldName}"); + break; + } + } + } + while (result is not null); + + return results; + } + + /// + /// Converts JSON objects to SPDX elements. + /// + /// + /// + public List ConvertToElements(List? jsonList, ref ParserResults results, string? requiredComplianceStandard, IReadOnlyCollection? entitiesWithDifferentNTIARequirements) + { + var elementsList = new List(); + + if (jsonList is null) + { + return elementsList; + } + + foreach (JObject jsonObject in jsonList) + { + var entityType = GetEntityType(jsonObject, requiredComplianceStandard, entitiesWithDifferentNTIARequirements); + + object? deserializedElement = null; + try + { + deserializedElement = JsonSerializer.Deserialize(jsonObject.ToString(), entityType); + } + catch (Exception e) + { + throw new ParserException(e.Message); + } + + if (deserializedElement != null) + { + elementsList.Add((Element)deserializedElement); + + switch (entityType?.Name) + { + case string name when name.Contains("File"): + results.FilesCount += 1; + break; + case string name when name.Contains("Package"): + results.PackagesCount += 1; + break; + case string name when name.Contains("ExternalMap"): + results.ReferencesCount += 1; + break; + case string name when name.Contains("Relationship"): + results.RelationshipsCount += 1; + break; + default: + Console.WriteLine($"Unrecognized entity type: {entityType?.Name}"); + break; + } + } + } + + // Validate if elements meet required compliance standards + switch (requiredComplianceStandard) + { + case "NTIA": + ValidateNTIARequirements(elementsList); + break; + case null: + Console.WriteLine("Invalid or no compliance standard specified."); + break; + default: + throw new ParserException("Invalid compliance standards"); + } + + return elementsList; + } + + public Type GetEntityType(JObject jsonObject, string? requiredComplianceStandard, IReadOnlyCollection? entitiesWithDifferentNTIARequirements) + { + var assembly = typeof(Element).Assembly; + var entityType = jsonObject["type"]?.ToString(); + + // For these special cases, remove the prefix from the type. + switch (entityType) + { + case "software_File": + entityType = "File"; + break; + case "software_Package": + entityType = "Package"; + break; + } + + // Validate properties based on required compliance standard. + if (requiredComplianceStandard != null && entitiesWithDifferentNTIARequirements != null && entitiesWithDifferentNTIARequirements.Contains(entityType)) + { + switch (requiredComplianceStandard) + { + case "NTIA": + entityType = "NTIA" + entityType; + break; + } + } + + var type = assembly.GetType($"Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities.{entityType}") ?? throw new ParserException($"{entityType} on {jsonObject} is invalid."); + + return type; + } + + public void ValidateNTIARequirements(List elementsList) + { + ValidateSbomDocCreationForNTIA(elementsList); + ValidateSbomFilesForNTIA(elementsList); + ValidateSbomPackagesForNTIA(elementsList); + } + + /// + /// Validate that information about the SBOM document is present. + /// + /// + /// + public void ValidateSbomDocCreationForNTIA(List elementsList) + { + var spdxDocumentElements = elementsList.Where(element => element is SpdxDocument); + if (spdxDocumentElements.Count() != 1) + { + throw new ParserException("SBOM document is not NTIA compliant because it must only contain one SpdxDocument element."); + } + + var spdxDocumentElement = spdxDocumentElements.First(); + + var spdxCreationInfoElement = (CreationInfo?)elementsList. + Where(element => element.Type == nameof(CreationInfo)). + FirstOrDefault(element => ((CreationInfo)element).Id == spdxDocumentElement.CreationInfoDetails) + ?? throw new ParserException($"SBOM document is not NTIA compliant because it must have a creationInfo element with ID of {spdxDocumentElement.CreationInfoDetails}"); + } + + /// + /// Validate that all files have declared and concluded licenses. + /// + /// + /// + public void ValidateSbomFilesForNTIA(List elementsList) + { + var fileElements = elementsList.Where(element => element is NTIAFile); + foreach (var fileElement in fileElements) + { + var fileSpdxId = fileElement.SpdxId; + + var fileHasSha256Hash = fileElement.VerifiedUsing. + Any(packageVerificationCode => packageVerificationCode.Algorithm == + Parsers.Spdx30SbomParser.Entities.Enums.HashAlgorithm.sha256); + + if (!fileHasSha256Hash) + { + throw new ParserException($"SBOM document is not NTIA compliant because file with SPDX ID {fileSpdxId} does not have a SHA256 hash."); + } + } + } + + /// + /// Validate that all packages have declared and concluded licenses. + /// + /// + /// + public void ValidateSbomPackagesForNTIA(List elementsList) + { + var packageElements = elementsList.Where(element => element is Package); + foreach (var packageElement in packageElements) + { + var packageSpdxId = packageElement.SpdxId; + + var packageHasSha256Hash = packageElement.VerifiedUsing. + Any(packageVerificationCode => packageVerificationCode.Algorithm == + Parsers.Spdx30SbomParser.Entities.Enums.HashAlgorithm.sha256); + + if (!packageHasSha256Hash) + { + throw new ParserException($"SBOM document is not NTIA compliant because package with SPDX ID {packageSpdxId} does not have a SHA256 hash."); + } + } + } + + /// + /// Sets metadata based on parsed SBOM elements. + /// + /// + public SpdxMetadata SetMetadata(ParserResults result) + { + var metadata = new SpdxMetadata(); + var spdxDocumentElement = (SpdxDocument?)result.FormatEnforcedSPDX3Result.Graph.FirstOrDefault(element => element.Type == "SpdxDocument"); + + if (spdxDocumentElement == null) + { + return metadata; + } + + if (spdxDocumentElement.NamespaceMap != null && spdxDocumentElement.NamespaceMap.TryGetValue("sbom", out var namespaceUri)) + { + metadata.DocumentNamespace = new Uri(namespaceUri); + } + + metadata.Name = spdxDocumentElement.Name; + metadata.SpdxId = spdxDocumentElement.SpdxId; + metadata.DocumentDescribes = spdxDocumentElement.RootElement; + + var dataLicenseSpdxId = spdxDocumentElement.DataLicense; + var spdxDataLicenseElement = result.FormatEnforcedSPDX3Result.Graph. + FirstOrDefault(element => element.SpdxId == dataLicenseSpdxId) as AnyLicenseInfo; + metadata.DataLicense = spdxDataLicenseElement?.Name; + + var spdxCreationInfoElement = (CreationInfo?)result.FormatEnforcedSPDX3Result.Graph. + Where(element => element.Type == nameof(CreationInfo)). + FirstOrDefault(element => ((CreationInfo)element).Id == spdxDocumentElement.CreationInfoDetails); + + var creators = spdxCreationInfoElement?.CreatedBy + .Select(createdBy => result.FormatEnforcedSPDX3Result.Graph. + FirstOrDefault(element => element.SpdxId == createdBy) as Organization) + .Where(spdxOrganizationElement => spdxOrganizationElement != null) + .Select(spdxOrganizationElement => spdxOrganizationElement?.Name) + .ToList() ?? []; + + creators.AddRange(spdxCreationInfoElement?.CreatedUsing + .Select(createdBy => result.FormatEnforcedSPDX3Result.Graph. + FirstOrDefault(element => element.SpdxId == createdBy) as Tool) + .Where(spdxToolElement => spdxToolElement != null) + .Select(spdxToolElement => spdxToolElement?.Name) ?? []); + + var createdDate = DateTime.MinValue; + DateTime.TryParse(spdxCreationInfoElement?.Created, out createdDate); + + metadata.CreationInfo = new MetadataCreationInfo + { + Created = createdDate, + Creators = creators, + }; + + metadata.SpdxVersion = spdxCreationInfoElement?.SpecVersion; + + return metadata; + } +} From 830788ed412e31f43bb3d8f229b86e0b2e60c12c Mon Sep 17 00:00:00 2001 From: ppandrate Date: Thu, 9 Jan 2025 20:31:40 -0800 Subject: [PATCH 2/4] Fix unit test syntax issues, clean up json parser --- .../LargeJsonParser.cs | 28 +- .../LargeJsonParser30.cs | 304 ------------------ .../Parser/SPDX30Parser.cs | 9 +- .../Parser/SbomMetadataParserTests.cs | 39 ++- .../Parser/SbomParserTestsBase.cs | 6 +- 5 files changed, 48 insertions(+), 338 deletions(-) delete mode 100644 src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/LargeJsonParser30.cs diff --git a/src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/LargeJsonParser.cs b/src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/LargeJsonParser.cs index a96de0f0..547964fd 100644 --- a/src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/LargeJsonParser.cs +++ b/src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/LargeJsonParser.cs @@ -9,6 +9,7 @@ using System.Text.Json; using System.Text.Json.Nodes; using Microsoft.Sbom.JsonAsynchronousNodeKit.Exceptions; +using Newtonsoft.Json.Linq; namespace Microsoft.Sbom.JsonAsynchronousNodeKit; @@ -33,18 +34,20 @@ public class LargeJsonParser private bool isParsingStarted = false; private bool enumeratorActive = false; private ParserStateResult? previousResultState = null; + private bool isSpdx30 = false; public LargeJsonParser( Stream stream, IReadOnlyDictionary handlers, JsonSerializerOptions? jsonSerializerOptions = default, - int bufferSize = DefaultReadBufferSize) + int bufferSize = DefaultReadBufferSize, + bool isSpdx30 = false) { this.stream = stream ?? throw new ArgumentNullException(nameof(stream)); this.handlers = handlers ?? throw new ArgumentNullException(nameof(handlers)); this.jsonSerializerOptions = jsonSerializerOptions ?? new JsonSerializerOptions(); - this.buffer = new byte[bufferSize]; + this.isSpdx30 = isSpdx30; // Validate buffer is not of 0 length. if (this.buffer.Length == 0) @@ -153,6 +156,7 @@ private ParserStateResult HandleExplicitProperty(ref Utf8JsonReader reader, stri var handler = this.handlers![propertyName]; object? result; + switch (handler.Type) { case ParameterType.String: @@ -268,11 +272,23 @@ private IEnumerable GetArray(Type type) private object GetObject(Type type, ref Utf8JsonReader reader) { - var jsonObject = ParserUtils.ParseObject(this.stream, ref this.buffer, ref reader); - object? result = jsonObject; - if (type != typeof(JsonNode)) + object? result = null; + + if (type == typeof(string)) + { + result = reader.GetString(); + } + else { - result = jsonObject.Deserialize(type, this.jsonSerializerOptions); + var jsonObject = ParserUtils.ParseObject(this.stream, ref this.buffer, ref reader); + if (type != typeof(JsonNode)) + { + result = jsonObject.Deserialize(type, this.jsonSerializerOptions); + } + else + { + result = jsonObject; + } } if (result is null) diff --git a/src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/LargeJsonParser30.cs b/src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/LargeJsonParser30.cs deleted file mode 100644 index dcd3677d..00000000 --- a/src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/LargeJsonParser30.cs +++ /dev/null @@ -1,304 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Linq.Expressions; -using System.Net.Http.Json; -using System.Text; -using System.Text.Json; -using System.Text.Json.Nodes; -using System.Text.Json.Serialization; -using System.Xml.Linq; -using Microsoft.Sbom.JsonAsynchronousNodeKit.Exceptions; -using Newtonsoft.Json; -using JsonException = System.Text.Json.JsonException; -using JsonSerializer = System.Text.Json.JsonSerializer; - -namespace Microsoft.Sbom.JsonAsynchronousNodeKit; - -#nullable enable - -/// -/// Allows for parsing large json objects without loading the entire object into memory. Large json arrays use a yield return to avoid having the whole enumerable in memory at once. -/// -/// -/// This class is not Thread-safe since the stream and JsonReaders assume a single forward-only reader. -/// Because of the use of recursion in the GetObject method, this class is also not suitable for parsing very deep json objects. -/// -public class LargeJsonParser30 -{ - private const int DefaultReadBufferSize = 4096; - private readonly Stream stream; - private readonly JsonSerializerOptions jsonSerializerOptions; - private byte[] buffer; - private JsonReaderState readerState; - private bool isFinalBlock; - private bool isParsingStarted = false; - private bool enumeratorActive = false; - private ParserStateResult? previousResultState = null; - private readonly Dictionary handlers; - - public LargeJsonParser30( - Stream stream, - Dictionary handlers, - JsonSerializerOptions? jsonSerializerOptions = default, - int bufferSize = DefaultReadBufferSize) - { - this.stream = stream ?? throw new ArgumentNullException(nameof(stream)); - this.handlers = handlers ?? throw new ArgumentNullException(nameof(handlers)); - this.jsonSerializerOptions = jsonSerializerOptions ?? new JsonSerializerOptions(); - - this.buffer = new byte[bufferSize]; - - // Validate buffer is not of 0 length. - if (this.buffer.Length == 0) - { - throw new ArgumentException($"The {nameof(this.buffer)} value can't be null or of 0 length."); - } - - this.stream.Position = GetStartPosition(this.stream); - - if (!stream.CanRead) - { - throw new NotSupportedException("The stream must be readable."); - } - - if (stream.Read(this.buffer) == 0) - { - throw new EndOfStreamException("No bytes were read from the stream."); - } - - static int GetStartPosition(Stream stream) - { - var bom = Encoding.UTF8.Preamble.ToArray(); - stream.Position = 0; - var buffer = new byte[bom.Length]; - _ = stream.Read(buffer, 0, buffer.Length); - - return Enumerable.SequenceEqual(buffer, bom) ? 3 : 0; - } - } - - /// - /// Begin evaluating the next section of the json stream. - /// - /// A ParserStateResult object describing the section that was encountered or null if this was the final state. - /// If the result object is an enumerable you MUST ensure that you've walked it entirely before calling next again. - public ParserStateResult? Next() - { - if (this.enumeratorActive) - { - throw new ParserException("You must walk the entire enumerable from the previous result before calling Next() again."); - } - - try - { - var reader = new Utf8JsonReader(this.buffer, isFinalBlock: this.isFinalBlock, this.readerState); - - if (!this.isParsingStarted) - { - ParserUtils.SkipNoneTokens(this.stream, ref this.buffer, ref reader); - - // Arrays are legal root objects, but if you expect that you should use `System.Text.Json.JsonSerializer.DeserializeAsyncEnumerable` instead. - if (reader.TokenType == JsonTokenType.StartArray) - { - throw new ParserException($"For root-level arrays use {nameof(JsonSerializer.DeserializeAsyncEnumerable)}."); - } - - ParserUtils.AssertTokenType(this.stream, ref reader, JsonTokenType.StartObject); - ParserUtils.GetMoreBytesFromStream(this.stream, ref this.buffer, ref reader); - - this.isParsingStarted = true; - } - - ParserUtils.Read(this.stream, ref this.buffer, ref reader); - - // If the end of the Json Object is reached, return null to indicate completion. - if (reader.TokenType == JsonTokenType.EndObject) - { - return null; - } - else if (this.previousResultState is not null && this.previousResultState.YieldReturn && reader.TokenType == JsonTokenType.EndArray) - { - // yield returning json arrays means we can't pass it the same Utf8JsonReader ref, so we need to create a new one. - // BUT when we do that we end up consuming the next token, so we need to leave it in the array case to be eaten by the next caller. - ParserUtils.Read(this.stream, ref this.buffer, ref reader); - } - - ParserUtils.AssertTokenType(this.stream, ref reader, JsonTokenType.PropertyName); - - var propertyName = reader.GetString() ?? throw new InvalidOperationException("It should not be possible to have a null PropertyName"); - ParserUtils.Read(this.stream, ref this.buffer, ref reader); - - var resultState = this.handlers.ContainsKey(propertyName) - ? this.HandleExplicitProperty(ref reader, propertyName) - : this.HandleExtraProperty(ref reader, propertyName); - - // yield return is a special case where we need to leave the reader in the same state as we found it. - if (!resultState.YieldReturn) - { - ParserUtils.GetMoreBytesFromStream(this.stream, ref this.buffer, ref reader); - - this.isFinalBlock = reader.IsFinalBlock; - this.readerState = reader.CurrentState; - } - - this.previousResultState = resultState; - - return resultState; - } - catch (JsonException ex) - { - throw new ParserException($"Error parsing json at position {this.stream.Position}", ex); - } - } - - private ParserStateResult HandleExplicitProperty(ref Utf8JsonReader reader, string propertyName) - { - var handler = this.handlers![propertyName]; - - object? result; - switch (handler.Type) - { - case ParameterType.String: - ParserUtils.AssertTokenType(this.stream, ref reader, JsonTokenType.String); - result = reader.GetString(); - break; - case ParameterType.Skip: - ParserUtils.SkipProperty(this.stream, ref this.buffer, ref reader); - result = null; - break; - case ParameterType.Int: - ParserUtils.AssertTokenType(this.stream, ref reader, JsonTokenType.Number); - var i = reader.GetInt32(); - result = i; - break; - case ParameterType.Object: - var objType = handler.GetType().GetGenericArguments()[0]; - result = this.GetObject(ref reader, objType); - break; - case ParameterType.Array: - var arrType = handler.GetType().GetGenericArguments()[0]; - result = this.ParseArray(ref reader, arrType); - break; - default: - throw new InvalidOperationException($"Unknown {nameof(ParameterType)}: {handler.Type}"); - } - - return new ParserStateResult(propertyName, result, ExplicitField: true, YieldReturn: handler.Type == ParameterType.Array); - } - - private ParserStateResult HandleExtraProperty(ref Utf8JsonReader reader, string propertyName) - { - object? result = reader.TokenType switch - { - JsonTokenType.String => reader.GetString(), - JsonTokenType.Number => reader.GetInt32(), - JsonTokenType.True => true, - JsonTokenType.False => false, - JsonTokenType.StartArray => ParserUtils.ParseArray(this.stream, ref this.buffer, ref reader), - JsonTokenType.StartObject => ParserUtils.ParseObject(this.stream, ref this.buffer, ref reader), - JsonTokenType.None => throw new NotImplementedException(), - JsonTokenType.EndObject => throw new NotImplementedException(), - JsonTokenType.EndArray => throw new NotImplementedException(), - JsonTokenType.PropertyName => throw new NotImplementedException(), - JsonTokenType.Comment => throw new NotImplementedException(), - JsonTokenType.Null => throw new NotImplementedException(), - _ => throw new InvalidOperationException($"Unknown {nameof(JsonTokenType)}: {reader.TokenType}"), - }; - return new ParserStateResult(propertyName, result, ExplicitField: false, YieldReturn: false); - } - - private IEnumerable ParseArray(ref Utf8JsonReader reader, Type objType) - { - ParserUtils.AssertTokenType(this.stream, ref reader, JsonTokenType.StartArray); - - // We can't pass the current reader along, so we need to save it's state for it. - ParserUtils.GetMoreBytesFromStream(this.stream, ref this.buffer, ref reader); - - this.isFinalBlock = reader.IsFinalBlock; - this.readerState = reader.CurrentState; - - return this.GetArray(objType); - } - - private IEnumerable GetArray(Type type) - { - this.enumeratorActive = true; - while (true) - { - var obj = this.ReadArrayObject(type); - - if (obj is not null) - { - yield return obj; - } - else - { - this.enumeratorActive = false; - yield break; - } - } - } - - private object? ReadArrayObject(Type type) - { - try - { - var reader = new Utf8JsonReader(this.buffer, this.isFinalBlock, this.readerState); - - ParserUtils.Read(this.stream, ref this.buffer, ref reader); - - object? result; - if (reader.TokenType == JsonTokenType.EndArray) - { - result = null; - } - else if (type.Name == "String") - { - result = reader.GetString(); - } - else - { - result = this.GetObject(ref reader, type); - } - - ParserUtils.GetMoreBytesFromStream(this.stream, ref this.buffer, ref reader); - - this.isFinalBlock = reader.IsFinalBlock; - this.readerState = reader.CurrentState; - - return result; - } - catch (JsonException ex) - { - throw new ParserException($"Error parsing json at position {this.stream.Position}", ex); - } - } - - private object GetObject(ref Utf8JsonReader reader, Type type) - { - var jsonObject = ParserUtils.ParseObject(this.stream, ref this.buffer, ref reader); - var jsonObjectType = jsonObject.AsObject()["type"]?.ToString(); - object? result; - - if (jsonObjectType == null) - { - throw new InvalidOperationException($"Deserializing {jsonObjectType} object type is not supported."); - } - else - { - result = JsonConvert.DeserializeObject(jsonObject.ToString()); - } - - if (result is null) - { - throw new InvalidOperationException($"Deserialization unexpectedly returned null."); - } - - return result; - } -} diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs index 0e9a3a5b..e03d5e13 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs @@ -38,7 +38,7 @@ public class SPDX30Parser : ISbomParser public string? RequiredComplianceStandard; public IReadOnlyCollection? EntitiesToEnforceComplianceStandardsFor; public SpdxMetadata Metadata = new SpdxMetadata(); - private readonly LargeJsonParser30 parser; + private readonly LargeJsonParser parser; private readonly IList observedFieldNames = new List(); private readonly bool requiredFieldsCheck = true; private readonly JsonSerializerOptions jsonSerializerOptions; @@ -70,23 +70,24 @@ public SPDX30Parser( var handlers = new Dictionary { { ContextProperty, new PropertyHandler(ParameterType.Array) }, - { GraphProperty, new PropertyHandler(ParameterType.Array) }, + { GraphProperty, new PropertyHandler(ParameterType.Array) }, }; switch (requiredComplianceStandard) { case "NTIA": this.EntitiesToEnforceComplianceStandardsFor = this.entitiesWithDifferentNTIARequirements; + this.RequiredComplianceStandard = requiredComplianceStandard; break; } if (bufferSize is null) { - this.parser = new LargeJsonParser30(stream, handlers, this.jsonSerializerOptions); + this.parser = new LargeJsonParser(stream, handlers, this.jsonSerializerOptions, isSpdx30: true); } else { - this.parser = new LargeJsonParser30(stream, handlers, this.jsonSerializerOptions, bufferSize.Value); + this.parser = new LargeJsonParser(stream, handlers, this.jsonSerializerOptions, bufferSize.Value, isSpdx30: true); } } diff --git a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomMetadataParserTests.cs b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomMetadataParserTests.cs index 14e08fd7..1eff3898 100644 --- a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomMetadataParserTests.cs +++ b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomMetadataParserTests.cs @@ -26,18 +26,18 @@ public void MetadataPopulates() var parser = new SPDX30Parser(stream); var results = this.Parse(parser); - Assert.IsTrue(results.FormatEnforcedSPDX3Result.Graph.Count() == 5); + Assert.AreEqual(5, results.FormatEnforcedSPDX3Result.Graph.Count()); var metadata = parser.GetMetadata(); Assert.IsNotNull(metadata); - Assert.IsTrue(metadata.DocumentNamespace != null); - Assert.IsTrue(metadata.Name == "spdx-doc-name"); - Assert.IsTrue(metadata.SpdxId == "SPDXRef-SpdxDocument-B93EED20C16A89A887B753958D42B794DD3C6570D3C2725B56B43477B38E05A1"); - Assert.IsTrue(metadata.DocumentDescribes.First() == "root-element-example"); - Assert.IsTrue(metadata.DataLicense == "CC0-1.0"); - Assert.IsTrue(metadata.CreationInfo.Creators.Count() == 2); - Assert.IsTrue(metadata.CreationInfo.Created != DateTime.MinValue); - Assert.IsTrue(metadata.SpdxVersion == "3.0"); + Assert.IsNotNull(metadata.DocumentNamespace); + Assert.AreEqual("spdx-doc-name", metadata.Name); + Assert.AreEqual("SPDXRef-SpdxDocument-B93EED20C16A89A887B753958D42B794DD3C6570D3C2725B56B43477B38E05A1", metadata.SpdxId); + Assert.AreEqual("root-element-example", metadata.DocumentDescribes.First()); + Assert.AreEqual("CC0-1.0", metadata.DataLicense); + Assert.AreEqual(2, metadata.CreationInfo.Creators.Count()); + Assert.AreNotEqual(DateTime.MinValue, metadata.CreationInfo.Created); + Assert.AreEqual("3.0", metadata.SpdxVersion); } [TestMethod] @@ -52,13 +52,13 @@ public void MetadataEmpty() var metadata = parser.GetMetadata(); Assert.IsNotNull(metadata); - Assert.IsTrue(metadata.DocumentNamespace == null); - Assert.IsTrue(metadata.Name == null); - Assert.IsTrue(metadata.SpdxId == null); - Assert.IsTrue(metadata.DocumentDescribes == null); - Assert.IsTrue(metadata.DataLicense == null); - Assert.IsTrue(metadata.CreationInfo == null); - Assert.IsTrue(metadata.SpdxVersion == null); + Assert.IsNull(metadata.DocumentNamespace); + Assert.IsNull(metadata.Name); + Assert.IsNull(metadata.SpdxId); + Assert.IsNull(metadata.DocumentDescribes); + Assert.IsNull(metadata.DataLicense); + Assert.IsNull(metadata.CreationInfo); + Assert.IsNull(metadata.SpdxVersion); } [TestMethod] @@ -79,7 +79,7 @@ public void DocCreation_NoName_Passes() var parser = new SPDX30Parser(stream); var results = this.Parse(parser); - Assert.IsTrue(results.FormatEnforcedSPDX3Result.Graph.Count() == 5); + Assert.AreEqual(5, results.FormatEnforcedSPDX3Result.Graph.Count()); var metadata = parser.GetMetadata(); Assert.IsNotNull(metadata); @@ -149,14 +149,11 @@ public void EmptyArray_ValidJson() } [TestMethod] - [ExpectedException(typeof(ArgumentException))] public void NullOrEmptyBuffer_Throws() { var bytes = Encoding.UTF8.GetBytes(SbomFullDocWithMetadataJsonStrings.MalformedJsonEmptyObject); using var stream = new MemoryStream(bytes); - var parser = new SPDX30Parser(stream, bufferSize: 0); - this.Parse(parser); - Assert.ThrowsException(() => this.Parse(parser)); + Assert.ThrowsException(() => new SPDX30Parser(stream, bufferSize: 0)); } } diff --git a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomParserTestsBase.cs b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomParserTestsBase.cs index 67a061ec..07dfb0e9 100644 --- a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomParserTestsBase.cs +++ b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomParserTestsBase.cs @@ -7,13 +7,13 @@ using System.Linq; using System.Linq.Expressions; using System.Text.Json; +using System.Text.Json.Nodes; using System.Text.Json.Serialization; using Microsoft.Sbom.Api.Entities.Output; using Microsoft.Sbom.Contracts; using Microsoft.Sbom.JsonAsynchronousNodeKit; using Microsoft.Sbom.JsonAsynchronousNodeKit.Exceptions; using Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities; -using Newtonsoft.Json.Linq; namespace Microsoft.Sbom.Parser; @@ -89,7 +89,7 @@ public List ConvertToElements(List? jsonList, ref ParserResults return elementsList; } - foreach (JObject jsonObject in jsonList) + foreach (JsonObject jsonObject in jsonList) { var entityType = GetEntityType(jsonObject, requiredComplianceStandard, entitiesWithDifferentNTIARequirements); @@ -144,7 +144,7 @@ public List ConvertToElements(List? jsonList, ref ParserResults return elementsList; } - public Type GetEntityType(JObject jsonObject, string? requiredComplianceStandard, IReadOnlyCollection? entitiesWithDifferentNTIARequirements) + public Type GetEntityType(JsonObject jsonObject, string? requiredComplianceStandard, IReadOnlyCollection? entitiesWithDifferentNTIARequirements) { var assembly = typeof(Element).Assembly; var entityType = jsonObject["type"]?.ToString(); From 5e5f17cee76a36f7fa6640677fe62136e0274eff Mon Sep 17 00:00:00 2001 From: ppandrate Date: Thu, 9 Jan 2025 21:01:43 -0800 Subject: [PATCH 3/4] remove unnecessary changes, clean up assemblies --- .../LargeJsonParser.cs | 1 - .../JsonAsynchronousNodeKit/ParserUtils.cs | 21 ------------------- .../Microsoft.Sbom.Common.csproj | 1 - .../Parser/SPDXParser.cs | 3 +-- .../Entities/Enums/RelationshipType.cs | 12 +++++------ .../Generator.cs | 8 ------- .../Parser/ExternalMapsResult.cs | 19 ----------------- .../Parser/FilesResult.cs | 19 ----------------- .../Parser/PackagesResult.cs | 19 ----------------- .../Parser/RelationshipsResult.cs | 19 ----------------- .../Utils/ElementSerializer.cs | 2 -- .../Parser/SbomFileParserTests.cs | 6 ------ .../Parser/SbomMetadataParserTests.cs | 2 -- .../Parser/SbomParserTestsBase.cs | 3 --- 14 files changed, 7 insertions(+), 128 deletions(-) delete mode 100644 src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/ExternalMapsResult.cs delete mode 100644 src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/FilesResult.cs delete mode 100644 src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/PackagesResult.cs delete mode 100644 src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/RelationshipsResult.cs diff --git a/src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/LargeJsonParser.cs b/src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/LargeJsonParser.cs index 547964fd..5794010f 100644 --- a/src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/LargeJsonParser.cs +++ b/src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/LargeJsonParser.cs @@ -9,7 +9,6 @@ using System.Text.Json; using System.Text.Json.Nodes; using Microsoft.Sbom.JsonAsynchronousNodeKit.Exceptions; -using Newtonsoft.Json.Linq; namespace Microsoft.Sbom.JsonAsynchronousNodeKit; diff --git a/src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/ParserUtils.cs b/src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/ParserUtils.cs index b13eafce..bb76e0b7 100644 --- a/src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/ParserUtils.cs +++ b/src/Microsoft.Sbom.Common/JsonAsynchronousNodeKit/ParserUtils.cs @@ -3,7 +3,6 @@ using System; using System.IO; -using System.Reflection.Metadata.Ecma335; using System.Text.Json; using System.Text.Json.Nodes; using Microsoft.Sbom.JsonAsynchronousNodeKit.Exceptions; @@ -210,26 +209,6 @@ internal static JsonObject ParseObject(Stream stream, ref byte[] buffer, ref Utf return node; } - internal static JsonNode ParseObjectAsString(Stream stream, ref byte[] buffer, ref Utf8JsonReader reader) - { - AssertTokenType(stream, ref reader, JsonTokenType.StartObject); - - JsonNode? value = null; - while (reader.TokenType != JsonTokenType.EndObject || value is JsonObject) - { - Read(stream, ref buffer, ref reader); - if (reader.TokenType == JsonTokenType.EndObject) - { - break; - } - - Read(stream, ref buffer, ref reader); - value = ParseValue(stream, ref buffer, ref reader); - } - - return value ?? throw new InvalidOperationException("Parsed value as string cannot be null."); - } - internal static JsonNode? ParseValue(Stream stream, ref byte[] buffer, ref Utf8JsonReader reader) => reader.TokenType switch { JsonTokenType.StartObject => ParseObject(stream, ref buffer, ref reader), diff --git a/src/Microsoft.Sbom.Common/Microsoft.Sbom.Common.csproj b/src/Microsoft.Sbom.Common/Microsoft.Sbom.Common.csproj index 174b0365..0c4e6b62 100644 --- a/src/Microsoft.Sbom.Common/Microsoft.Sbom.Common.csproj +++ b/src/Microsoft.Sbom.Common/Microsoft.Sbom.Common.csproj @@ -9,7 +9,6 @@ - diff --git a/src/Microsoft.Sbom.Parsers.Spdx22SbomParser/Parser/SPDXParser.cs b/src/Microsoft.Sbom.Parsers.Spdx22SbomParser/Parser/SPDXParser.cs index b8f8747c..53a4b248 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx22SbomParser/Parser/SPDXParser.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx22SbomParser/Parser/SPDXParser.cs @@ -162,8 +162,7 @@ internal SPDXParser( { if (!this.observedFieldNames.Contains(requiredField)) { - // TODO: remove this commented out part - // throw new ParserException($"Required field {requiredField} was not found in the SPDX file"); + throw new ParserException($"Required field {requiredField} was not found in the SPDX file"); } } } diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/Enums/RelationshipType.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/Enums/RelationshipType.cs index 6a4e90b7..9f0a3b8b 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/Enums/RelationshipType.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/Enums/RelationshipType.cs @@ -43,7 +43,7 @@ public enum RelationshipType CONTAINS, /// - /// The from Vulnerability is coordinatedBy the to Agent(ContextsResult) (vendor, researcher, or consumer agent). + /// The from Vulnerability is coordinatedBy the to Agent(s) (vendor, researcher, or consumer agent). /// COORDINATED_BY, @@ -69,7 +69,7 @@ public enum RelationshipType DESCENDANT_OF, /// - /// The from Element describes each to Element. To denote the root(ContextsResult) of a tree of elements in a collection, the rootElement property should be used. + /// The from Element describes each to Element. To denote the root(s) of a tree of elements in a collection, the rootElement property should be used. /// DESCRIBES, @@ -89,7 +89,7 @@ public enum RelationshipType EXPLOIT_CREATED_BY, /// - /// Designates a from Vulnerability has been fixed by the to Agent(ContextsResult). + /// Designates a from Vulnerability has been fixed by the to Agent(s). /// FIXED_BY, @@ -99,7 +99,7 @@ public enum RelationshipType FIXED_IN, /// - /// Designates a from Vulnerability was originally discovered by the to Agent(ContextsResult). + /// Designates a from Vulnerability was originally discovered by the to Agent(s). /// FOUND_BY, @@ -294,12 +294,12 @@ public enum RelationshipType SERIALIZED_IN_ARTIFACT, /// - /// The from Element has been tested on the to Element(ContextsResult). + /// The from Element has been tested on the to Element(s). /// TESTED_ON, /// - /// The from Element has been trained on the to Element(ContextsResult). + /// The from Element has been trained on the to Element(s). /// TRAINED_ON, diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Generator.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Generator.cs index e91f3ef0..660c6c53 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Generator.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Generator.cs @@ -3,22 +3,14 @@ using System; using System.Collections.Generic; -using System.ComponentModel; -using System.Dynamic; -using System.IO; using System.Linq; -using System.Reflection; using System.Text; using System.Text.Json; -using System.Text.Json.Serialization; -using System.Threading.Tasks.Sources; -using System.Xml.Linq; using Microsoft.Sbom.Common; using Microsoft.Sbom.Contracts; using Microsoft.Sbom.Contracts.Enums; using Microsoft.Sbom.Extensions; using Microsoft.Sbom.Extensions.Entities; -using Microsoft.Sbom.Parsers.Spdx30SbomParser; using Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities; using Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities.Enums; using Microsoft.Sbom.Parsers.Spdx30SbomParser.Exceptions; diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/ExternalMapsResult.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/ExternalMapsResult.cs deleted file mode 100644 index bca7846e..00000000 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/ExternalMapsResult.cs +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using System.Collections.Generic; -using System.Linq; -using Microsoft.Sbom.JsonAsynchronousNodeKit; -using Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities; - -namespace Microsoft.Sbom.Parser; - -public record class ExternalMapsResult : ParserStateResult -{ - public ExternalMapsResult(ParserStateResult result) - : base(result.FieldName, result.Result, result.ExplicitField, result.YieldReturn) - { - } - - public IEnumerable References => ((IEnumerable)this.Result!).Select(r => (ExternalMap)r); -} diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/FilesResult.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/FilesResult.cs deleted file mode 100644 index b3e9e0aa..00000000 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/FilesResult.cs +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using System.Collections.Generic; -using System.Linq; -using Microsoft.Sbom.JsonAsynchronousNodeKit; -using Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities; - -namespace Microsoft.Sbom.Parser; - -public record FilesResult : ParserStateResult -{ - public FilesResult(ParserStateResult result) - : base(result.FieldName, result.Result, result.ExplicitField, result.YieldReturn) - { - } - - public IEnumerable Files => ((IEnumerable)this.Result!).Select(r => (File)r); -} diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/PackagesResult.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/PackagesResult.cs deleted file mode 100644 index 9c3aa85b..00000000 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/PackagesResult.cs +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using System.Collections.Generic; -using System.Linq; -using Microsoft.Sbom.JsonAsynchronousNodeKit; -using Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities; - -namespace Microsoft.Sbom.Parser; - -public record PackagesResult : ParserStateResult -{ - public PackagesResult(ParserStateResult result) - : base(result.FieldName, result.Result, result.ExplicitField, result.YieldReturn) - { - } - - public IEnumerable Packages => ((IEnumerable)this.Result!).Select(r => (Package)r); -} diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/RelationshipsResult.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/RelationshipsResult.cs deleted file mode 100644 index 3520b5f3..00000000 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/RelationshipsResult.cs +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using System.Collections.Generic; -using System.Linq; -using Microsoft.Sbom.JsonAsynchronousNodeKit; -using Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities; - -namespace Microsoft.Sbom.Parser; - -public record RelationshipsResult : ParserStateResult -{ - public RelationshipsResult(ParserStateResult result) - : base(result.FieldName, result.Result, result.ExplicitField, result.YieldReturn) - { - } - - public IEnumerable Relationships => ((IEnumerable)this.Result!).Select(r => (Relationship)r); -} diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Utils/ElementSerializer.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Utils/ElementSerializer.cs index 145ea62e..f6c06f8b 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Utils/ElementSerializer.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Utils/ElementSerializer.cs @@ -5,8 +5,6 @@ using System.Collections.Generic; using System.Text.Json; using System.Text.Json.Serialization; -using System.Xml.Linq; -using Microsoft.Sbom.Contracts; using Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities; public class ElementSerializer : JsonConverter> diff --git a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomFileParserTests.cs b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomFileParserTests.cs index c13a1fde..40d6e0ea 100644 --- a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomFileParserTests.cs +++ b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomFileParserTests.cs @@ -1,17 +1,11 @@ // Copyright (c) Microsoft. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System; using System.IO; -using System.Linq; using System.Text; using Microsoft.Sbom.JsonAsynchronousNodeKit.Exceptions; using Microsoft.Sbom.Parser.JsonStrings; -using Microsoft.Sbom.Parsers.Spdx30SbomParser; -using Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities; -using Microsoft.Sbom.Utils; using Microsoft.VisualStudio.TestTools.UnitTesting; -using File = Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities.File; namespace Microsoft.Sbom.Parser; diff --git a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomMetadataParserTests.cs b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomMetadataParserTests.cs index 1eff3898..2c264632 100644 --- a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomMetadataParserTests.cs +++ b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomMetadataParserTests.cs @@ -2,14 +2,12 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; -using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; using Microsoft.Sbom.JsonAsynchronousNodeKit.Exceptions; using Microsoft.Sbom.Parser.JsonStrings; using Microsoft.Sbom.Parsers.Spdx30SbomParser; -using Microsoft.Sbom.Utils; using Microsoft.VisualStudio.TestTools.UnitTesting; namespace Microsoft.Sbom.Parser; diff --git a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomParserTestsBase.cs b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomParserTestsBase.cs index 07dfb0e9..bc682214 100644 --- a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomParserTestsBase.cs +++ b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomParserTestsBase.cs @@ -5,11 +5,8 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Linq.Expressions; using System.Text.Json; using System.Text.Json.Nodes; -using System.Text.Json.Serialization; -using Microsoft.Sbom.Api.Entities.Output; using Microsoft.Sbom.Contracts; using Microsoft.Sbom.JsonAsynchronousNodeKit; using Microsoft.Sbom.JsonAsynchronousNodeKit.Exceptions; From dadcb299bed899c2e86d4a8a8ba169a7ccef67ec Mon Sep 17 00:00:00 2001 From: ppandrate Date: Thu, 9 Jan 2025 21:51:51 -0800 Subject: [PATCH 4/4] fix bugs in generator tests --- .../Parser/JsonStrings/SbomDocCreationJsonStrings.cs | 12 ++++-------- .../Parser/JsonStrings/SbomFileJsonStrings.cs | 4 ++-- .../Parser/JsonStrings/SbomPackageJsonStrings.cs | 8 ++++---- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/JsonStrings/SbomDocCreationJsonStrings.cs b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/JsonStrings/SbomDocCreationJsonStrings.cs index 1394e2c4..a1832056 100644 --- a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/JsonStrings/SbomDocCreationJsonStrings.cs +++ b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/JsonStrings/SbomDocCreationJsonStrings.cs @@ -42,14 +42,10 @@ public static class SbomDocCreationJsonStrings }, { ""dataLicense"": ""SPDXRef-AnyLicenseInfo-6E237C55B0583CB7BBA05562316C54B0A105ABA04775017E2253237B9A64613C"", - ""namespaceMap"": [ - { - ""namespace"": ""http://sbom.microsoft/sbom-package-name/sbom-package-version/some-custom-value-here"", - ""creationInfo"": ""_:creationinfo"", - ""spdxId"": ""SPDXRef-NamespaceMap-0C5D68EB49795A98E060EB263AC73F87322217857EB3057EBAC84A70F75E69BE"", - ""type"": ""NamespaceMap"" - } - ], + ""namespaceMap"": + { + ""sbom"":""http://sbom.microsoft/sbom-package-name/sbom-package-version/some-custom-value-here"" + }, ""profileConformance"": [ ""software"", ""core"", diff --git a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/JsonStrings/SbomFileJsonStrings.cs b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/JsonStrings/SbomFileJsonStrings.cs index c05bc2a9..d311a347 100644 --- a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/JsonStrings/SbomFileJsonStrings.cs +++ b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/JsonStrings/SbomFileJsonStrings.cs @@ -9,9 +9,9 @@ public static class SbomFileJsonStrings public const string FileWithLicensesAndHashes = @"[ { - ""CopyrightText"": ""sampleCopyright"", - ""creationInfo"": ""_:creationinfo"", ""name"": ""./sample/path"", + ""software_copyrightText"": ""sampleCopyright"", + ""creationInfo"": ""_:creationinfo"", ""spdxId"": ""SPDXRef-software_File-B4A9F99A3A03B9273AE34753D96564CB4F2B0FAD885BBD36B0DD619E9E8AC967"", ""verifiedUsing"": [ { diff --git a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/JsonStrings/SbomPackageJsonStrings.cs b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/JsonStrings/SbomPackageJsonStrings.cs index 15f89746..b949a9e8 100644 --- a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/JsonStrings/SbomPackageJsonStrings.cs +++ b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/JsonStrings/SbomPackageJsonStrings.cs @@ -16,13 +16,13 @@ public static class SbomPackageJsonStrings }, { ""suppliedBy"": ""SPDXRef-Organization-8560FC6692684D8DF52223FF78E30B9630A1CF5A6FA371AAE24FCA896AE20969"", - ""CopyrightText"": ""NOASSERTION"", + ""name"": ""test"", + ""software_copyrightText"": ""NOASSERTION"", ""software_downloadLocation"": ""NOASSERTION"", ""creationInfo"": ""_:creationinfo"", ""externalIdentifier"": [ ""SPDXRef-ExternalIdentifier-CE6B7E4A59503594D0AF89492494E05071399F36D9085F1E722497D78A9404E1"" ], - ""name"": ""test"", ""spdxId"": ""SPDXRef-software_Package-4739C82D88855A138C811B8CE05CC97113BEC7F7C7F66EC7E4C6C176EEA0FECE"", ""type"": ""software_Package"" }, @@ -84,14 +84,14 @@ public static class SbomPackageJsonStrings ""type"": ""Organization"" }, { - ""CopyrightText"": ""NOASSERTION"", + ""name"": ""the-package-name"", + ""software_copyrightText"": ""NOASSERTION"", ""software_downloadLocation"": ""NOASSERTION"", ""software_packageVersion"": ""the-package-version"", ""creationInfo"": ""_:creationinfo"", ""externalIdentifier"": [ ""SPDXRef-ExternalIdentifier-.*"" ], - ""name"": ""the-package-name"", ""spdxId"": ""SPDXRef-software_Package-A8C9BE15D102D0AF9D34A9EAA6FE282AB0CE35FF48A83058703A964E512B68B7"", ""verifiedUsing"": [ {