Merge branch 'develop' into main

Author: JeroenKnoops

README.md

@@ -1,4 +1,3 @@
-![Coverage](coverage_badge.svg)
 # Bompare, a tool to compare the Software Bill-of-Materials from multiple sources
 
 ## Usage
@@ -31,6 +30,13 @@ E.g.:
     - OSX (Mac) using brew: `brew tap dart-lang/dart` and then `brew install dart`
     - Windows using [Chocolatey](https://chocolatey.org): `choco install dart-sdk`
     - With docker ` docker run -it --rm -v $(pwd):/work -w /work google/dart ./build.sh`
-2. Run `build.sh` to run all tests and build a native executable
-called `bompare`.
+1. Globally install the coverage helper tooling: `dart pub global activate coverage`.
+1. Globally install the flutter_coverage_badge: `dart pub global activate flutter_coverage_badge`.
+1. Install "lcov" coverage visualization tooling.
+1. Run `build.sh` to run all tests and build a native executable
+   called `bompare`.
+
+If the coverage tools are installed, the build results in an update of the
+coverage badge and a [static web site](coverage/index.html) with coverage
+details.
 

analysis_options.yaml

@@ -1,14 +1 @@
-# Defines a default set of lint rules enforced for
-# projects at Google. For details and rationale,
-# see https://github.com/dart-lang/pedantic#enabled-lints.
-include: package:pedantic/analysis_options.yaml
-
-# For lint rules and documentation, see http://dart-lang.github.io/linter/lints.
-# Uncomment to specify additional rules.
-# linter:
-#   rules:
-#     - camel_case_types
-
-analyzer:
-#   exclude:
-#     - path/to/excluded/files/**
+include: package:lints/recommended.yaml

build.sh

@@ -11,21 +11,14 @@ dart analyze "test"
 dart format "bin" "lib" "test"
 
 # Calculate coverage by unit tests
-dart run test test --coverage coverage
+dart run test --coverage coverage
 
-if hash format_coverage >/dev/null && hash genhtml >/dev/null; then
-  format_coverage --lcov --in=coverage --out=coverage/lcov.info --packages=.packages --report-on=lib
+if type format_coverage && type genhtml &> /dev/null ; then
+  dart run coverage:format_coverage --lcov --in=coverage --out=coverage/lcov.info --packages=.packages --report-on=lib
   genhtml -o coverage coverage/lcov.info
-  echo "Coverage report is found in /coverage/index.html"
-
-  if hash flutter_coverage_badge >/dev/null; then
-    flutter_coverage_badge
-    echo "Coverage badge generated."
-  else
-    echo "(Install dart package:flutter_coverage_badge globally to generate coverage badge.)"
-  fi
+  echo "Coverage report is found in ./coverage/index.html"
 else
-  echo "(Install dart package:coverage globally and genhtml to generate a full coverage overview.)"
+  echo "(Install dart package:coverage globally and lcov (genhtml) to generate a full coverage overview.)"
 fi
 
 # Build command line executable

coverage_badge.svg

@@ -6,6 +6,7 @@
 import 'dart:io';
 
 import 'package:args/command_runner.dart';
+import 'package:bompare/service/purl.dart';
 import 'package:glob/glob.dart';
 
 import '../service/bom_service.dart';
@@ -20,6 +21,7 @@ abstract class AbstractCommand extends Command {
   static const option_spdx = 'spdx-tag-value';
   static const option_white_source = 'whitesource';
   static const option_black_duck = 'blackduck';
+  static const option_default_type = 'default-type';
   static const option_output = 'out';
   static const option_diff_only = 'diffOnly';
   static const option_verbose = 'verbose';
@@ -56,6 +58,9 @@ abstract class AbstractCommand extends Command {
           abbr: 'b',
           help: 'Scan result in Black Duck "report" (ZIP/directory) format',
           valueHelp: 'filename or directory glob pattern')
+      ..addOption(option_default_type,
+          help: 'Overrides the default package type (from "generic")',
+          valueHelp: 'type')
       ..addOption(option_spdx_mapping,
           help: 'Convert license texts using SPDX mapping CSV file, '
               'formatted as: "<license text>","<SPDX identifier>"',
@@ -94,6 +99,11 @@ abstract class AbstractCommand extends Command {
       await service.loadSpdxMapping(File(spdxMapping));
     }
 
+    final defaultType = argResults![option_default_type];
+    if (defaultType != null) {
+      Purl.defaultType = defaultType;
+    }
+
     await Future.wait([
       _loadTypedResults(option_reference, ScannerType.reference),
       _loadTypedResults(

lib/command/abstract_command.dart

@@ -9,9 +9,10 @@ import 'dart:io';
 import 'package:archive/archive.dart';
 import 'package:path/path.dart' as path;
 
-import '../../service/domain/item_id.dart';
+import '../../service/domain/bom_item.dart';
 import '../../service/domain/scan_result.dart';
 import '../../service/domain/spdx_mapper.dart';
+import '../../service/purl.dart';
 import '../persistence_exception.dart';
 import '../result_parser.dart';
 import 'csv_parser.dart';
@@ -103,17 +104,18 @@ class BlackDuckResultParser implements ResultParser {
       stream.transform(utf8.decoder).transform(LineSplitter());
 }
 
-/// Dictionary to lookup licenses per [ItemId].
+/// Dictionary to lookup licenses per [BomItem].
 class _LicenseDictionary {
-  final _dict = <ItemId, Set<String>>{};
+  /// Version id -> Licenses
+  final _dict = <String, Set<String>>{};
 
-  void addLicenses(ItemId id, Iterable<String> values) {
-    final licenses = _dict[id] ?? {};
+  void addLicenses(String versionId, Iterable<String> values) {
+    final licenses = _dict[versionId] ?? {};
     licenses.addAll(values);
-    _dict[id] = licenses;
+    _dict[versionId] = licenses;
   }
 
-  Set<String>? operator [](ItemId id) => _dict[id];
+  Set<String>? operator [](String versionId) => _dict[versionId];
 }
 
 /// Extracts license info per component from the Black Duck components CSV file.
@@ -122,40 +124,37 @@ class _BlackDuckComponentsCsvParser extends CsvParser {
   final _LicenseDictionary _dictionary;
   final SpdxMapper mapper;
 
-  var _componentNameIndex = -1;
-  var _componentVersionIndex = -1;
-  var _licensesIndex = -1;
+  late final int _licensesIndex;
+  late final int _componentVersionIdIndex;
 
   _BlackDuckComponentsCsvParser(this._dictionary, this.mapper);
 
   @override
   void headerRow(List<String> columns) {
-    _componentNameIndex = columnIndexOf('Component name', columns);
-    _componentVersionIndex = columnIndexOf('Component version name', columns);
     _licensesIndex = columnIndexOf('License names', columns);
+    _componentVersionIdIndex = columnIndexOf('Version id', columns);
   }
 
   @override
   void dataRow(List<String> columns) {
-    final component = columns[_componentNameIndex];
-    final version = columns[_componentVersionIndex];
-    final id = ItemId(component, version);
     final license = columns[_licensesIndex];
-    _dictionary.addLicenses(id, mapper[license]);
+    final versionId = columns[_componentVersionIdIndex];
+    _dictionary.addLicenses(versionId, mapper[license]);
   }
 }
 
 /// Extracts dependencies from a Black Duck source CSV.
 class _BlackDuckSourceCsvParser extends CsvParser {
   final ScanResult result;
   final _LicenseDictionary licenseDictionary;
-  final assumed = <ItemId>{};
+  final assumed = <BomItem>{};
 
-  var _versionIndex = -1;
-  var _originIndex = -1;
-  var _nameIndex = -1;
-  var _componentNameIndex = -1;
-  var _componentVersionIndex = -1;
+  late final int _versionIndex;
+  late final int _originIndex;
+  late final int _nameIndex;
+  late final int _componentNameIndex;
+  late final int _componentVersionIndex;
+  late final int _componentVersionIdIndex;
 
   _BlackDuckSourceCsvParser(this.result, this.licenseDictionary);
 
@@ -166,62 +165,68 @@ class _BlackDuckSourceCsvParser extends CsvParser {
     _nameIndex = columnIndexOf('Origin name id', columns);
     _componentNameIndex = columnIndexOf('Component name', columns);
     _componentVersionIndex = columnIndexOf('Component version name', columns);
+    _componentVersionIdIndex = columnIndexOf('Version id', columns);
   }
 
   @override
   void dataRow(List<String> columns) {
-    final type = columns[_originIndex];
+    final origin = columns[_originIndex];
     final nameColumn = columns[_nameIndex];
     final versionColumn = columns[_versionIndex];
+    final versionId = columns[_componentVersionIdIndex];
+    final type = _purlType[origin] ?? origin;
 
-    var itemId;
-    switch (type) {
+    late BomItem item;
+    switch (origin) {
       case '': // Signature scan result
         final component = columns[_componentNameIndex];
         final componentVersion = columns[_componentVersionIndex];
-        itemId = ItemId(component, componentVersion);
+        item = BomItem(Purl.of(
+            type: 'generic', name: component, version: componentVersion));
         break;
       case 'maven':
       case 'github':
         final name2 = _stripFromLast(nameColumn, ':').replaceAll(':', '/');
-        itemId = ItemId(name2, versionColumn);
+        item =
+            BomItem(Purl.of(type: type, name: name2, version: versionColumn));
         break;
       case 'npmjs':
       case 'nuget':
         final name2 = _stripFromLast(nameColumn, '/');
-        itemId = ItemId(name2, versionColumn);
+        item =
+            BomItem(Purl.of(type: type, name: name2, version: versionColumn));
         break;
       case 'alpine':
         final name = _stripSeparatedPostFix(nameColumn, versionColumn);
         final version = _stripFromLast(versionColumn, '/');
-        itemId = ItemId(name, version);
+        item = BomItem(Purl.of(type: type, name: name, version: version));
         break;
       case 'centos':
         final name = _stripFrom(nameColumn, '/');
         final temp = versionColumn.substring(versionColumn.indexOf(':') + 1);
         final version = _stripFrom(temp, '-');
-        itemId = ItemId(name, version);
+        item = BomItem(Purl.of(type: type, name: name, version: version));
         break;
       case 'debian':
         final version = _stripFrom(versionColumn, '/');
         final name = _stripSeparatedPostFix(nameColumn, version);
-        itemId = ItemId(name, version);
+        item = BomItem(Purl.of(type: type, name: name, version: version));
         break;
       case 'long_tail':
         final name = _stripFromLast(nameColumn, '#');
-        itemId = ItemId(name, versionColumn);
+        item = BomItem(Purl.of(type: type, name: name, version: versionColumn));
         break;
       default:
         final name = _stripFromLast(nameColumn, '/');
-        itemId = ItemId(name, versionColumn);
-        if (!assumed.contains(itemId)) {
+        item = BomItem(Purl.of(type: type, name: name, version: versionColumn));
+        if (!assumed.contains(item)) {
           print(
-              'Warning: Assumed name=${itemId.package}, version=${itemId.version} for Black Duck type "$type"');
-          assumed.add(itemId);
+              'Warning: Assumed ${item.purl} for Black Duck type "$origin" -> "$nameColumn"');
+          assumed.add(item);
         }
     }
-    _addLicenses(columns, itemId);
-    result.addItem(itemId);
+    item.addLicenses(licenseDictionary[versionId] ?? {});
+    result.addItem(item);
   }
 
   String _stripFrom(String string, Pattern pattern) {
@@ -238,11 +243,19 @@ class _BlackDuckSourceCsvParser extends CsvParser {
     final index = string.lastIndexOf(postfix);
     return (index < 0) ? string : string.substring(0, index - 1);
   }
-
-  void _addLicenses(List<String> columns, ItemId itemId) {
-    final componentName = columns[_componentNameIndex];
-    final componentVersion = columns[_componentVersionIndex];
-    final component = ItemId(componentName, componentVersion);
-    itemId.addLicenses(licenseDictionary[component] ?? {});
-  }
 }
+
+const _purlType = {
+  'arch_linux': 'arch',
+  'centos': 'rpm',
+  'fedora': 'rpm',
+  'redhat': 'rpm',
+  'opensuse': 'rpm',
+  'crates': 'cargo',
+  'dart': 'pub',
+  'debian': 'deb',
+  'ubuntu': 'deb',
+  'long_tail': 'generic',
+  'npmjs': 'npm',
+  'rubygems': 'gem',
+};

lib/persistence/parser/blackduck_result_parser.dart

@@ -8,9 +8,10 @@ import 'dart:io';
 
 import 'package:path/path.dart' as path;
 
-import '../../service/domain/item_id.dart';
+import '../../service/domain/bom_item.dart';
 import '../../service/domain/scan_result.dart';
 import '../../service/domain/spdx_mapper.dart';
+import '../../service/purl.dart';
 import '../persistence_exception.dart';
 import '../result_parser.dart';
 
@@ -40,7 +41,8 @@ class Jk1ResultParser implements ResultParser {
         final package = (obj[field_module_name] as String).replaceAll(':', '/');
         final version = obj[field_module_version];
         var license = obj[field_module_license];
-        return ItemId(package, version)..addLicenses(mapper[license]);
+        return BomItem(Purl.of(type: 'maven', name: package, version: version))
+          ..addLicenses(mapper[license]);
       }).forEach((id) => result.addItem(id));
 
       return Future.value(result);

lib/persistence/parser/jk1_result_parser.dart

@@ -8,9 +8,10 @@ import 'dart:io';
 
 import 'package:path/path.dart' as path;
 
-import '../../service/domain/item_id.dart';
+import '../../service/domain/bom_item.dart';
 import '../../service/domain/scan_result.dart';
 import '../../service/domain/spdx_mapper.dart';
+import '../../service/purl.dart';
 import '../persistence_exception.dart';
 import '../result_parser.dart';
 import 'csv_parser.dart';
@@ -60,7 +61,8 @@ class _LicenseFileParser extends CsvParser {
     final version = module.substring(pos + 1);
     final name = module.substring(0, pos);
     final licenses = mapper[columns[_license_column]];
-    final item = ItemId(name, version)..addLicenses(licenses);
+    final item = BomItem(Purl.of(type: 'npm', name: name, version: version))
+      ..addLicenses(licenses);
     result.addItem(item);
   }