Merge pull request #53 from sw360/have_more_type_support

Have more type support

.gitignore

@@ -28,10 +28,7 @@ htmlcov/
 coverage.xml
 
 # preliminary
-sw360/
 cli/
-xxsw360/
-xxcli/
 Releases/
 
 # internal stuff

ChangeLog.md

@@ -5,6 +5,13 @@
 
 # CaPyCli - Clearing Automation Python Command Line Tool for SW360
 
+## 2.2.0 (2024-01-28)
+
+* `getdependencies javascript` can now handle package-lock.json files of version 3.
+* `bom findsources` can do source URL discovery using sw360 lookup, perform extensive
+  GitLab deep search, and adapt search strategy based on diverse programming languages.
+* Have type support.
+
 ## 2.1.0 (2023-12-16)
 
 * Be more resilient about missing metadata in CycloneDX SBOMs.

RunChecks.ps1

@@ -13,7 +13,12 @@ Write-Host "markdownlint ..."
 npx -q markdownlint-cli *.md --disable MD041
 
 Write-Host "isort ..."
-isort .
+poetry run isort .
+
+Write-Host "mypy ..."
+poetry run mypy .
+
+Write-Host "Done."
 
 # -----------------------------------
 # -----------------------------------

capycli/bom/bom_convert.py

@@ -13,6 +13,8 @@
 from enum import Enum
 from typing import Any
 
+from sortedcontainers import SortedSet
+
 import capycli.common.json_support
 import capycli.common.script_base
 from capycli import get_logger
@@ -58,23 +60,23 @@ def convert(self,
             # default is CaPyCLI
             outputformat = BomFormat.CAPYCLI
 
-        cdx_components = []
+        cdx_components: SortedSet
         project = None
         sbom = None
         try:
             if inputformat == BomFormat.TEXT:
-                cdx_components = PlainTextSupport.flatlist_to_cdx_components(inputfile)
+                cdx_components = SortedSet(PlainTextSupport.flatlist_to_cdx_components(inputfile))
                 print_text(f"  {len(cdx_components)} components read from file {inputfile}")
             elif inputformat == BomFormat.CSV:
-                cdx_components = CsvSupport.csv_to_cdx_components(inputfile)
+                cdx_components = SortedSet(CsvSupport.csv_to_cdx_components(inputfile))
                 print_text(f"  {len(cdx_components)} components read from file {inputfile}")
             elif (inputformat == BomFormat.CAPYCLI) or (inputformat == BomFormat.SBOM):
                 sbom = CaPyCliBom.read_sbom(inputfile)
                 cdx_components = sbom.components
                 project = sbom.metadata.component
                 print_text(f"  {len(cdx_components)} components read from file {inputfile}")
             elif inputformat == BomFormat.LEGACY:
-                cdx_components = LegacySupport.legacy_to_cdx_components(inputfile)
+                cdx_components = SortedSet(LegacySupport.legacy_to_cdx_components(inputfile))
                 print_text(f"  {len(cdx_components)} components read from file {inputfile}")
             elif inputformat == BomFormat.LEGACY_CX:
                 sbom = LegacyCx.read_sbom(inputfile)
@@ -89,7 +91,7 @@ def convert(self,
 
         try:
             if outputformat == BomFormat.TEXT:
-                PlainTextSupport.write_cdx_components_as_flatlist(cdx_components, outputfile)
+                PlainTextSupport.write_cdx_components_as_flatlist2(cdx_components, outputfile)
                 print_text(f"  {len(cdx_components)} components written to file {outputfile}")
             elif outputformat == BomFormat.CSV:
                 CsvSupport.write_cdx_components_as_csv(cdx_components, outputfile)
@@ -147,7 +149,7 @@ def display_help(self) -> None:
         print("    -if INPUTFORMAT       Specify input file format: capycli|sbom|text|csv|legacy|legacy-cx")
         print("    -of OUTPUTFORMAT      Specify output file format: capycli|text|csv|legacy|html")
 
-    def run(self, args):
+    def run(self, args: Any) -> None:
         """Main method()"""
         print("\n" + capycli.APP_NAME + ", " + capycli.get_app_version() + " - Convert SBOM formats\n")
 

capycli/bom/check_bom.py

@@ -1,5 +1,5 @@
 # -------------------------------------------------------------------------------
-# Copyright (c) 2019-23 Siemens
+# Copyright (c) 2019-2024 Siemens
 # All Rights Reserved.
 # Author: thomas.graf@siemens.com
 #
@@ -9,12 +9,13 @@
 import logging
 import os
 import sys
+from typing import Any, Dict, Optional
 
 import requests
-import sw360.sw360_api
 from colorama import Fore, Style
 from cyclonedx.model.bom import Bom
 from cyclonedx.model.component import Component
+from sw360 import SW360Error
 
 import capycli.common.script_base
 from capycli.common.capycli_bom_support import CaPyCliBom, CycloneDxSupport
@@ -39,57 +40,73 @@ def _bom_has_items_without_id(self, bom: Bom) -> bool:
 
         return False
 
-    def _find_by_id(self, component: Component) -> dict:
+    def _find_by_id(self, component: Component) -> Optional[Dict[str, Any]]:
+        if not self.client:
+            print_red("  No client!")
+            sys.exit(ResultCode.RESULT_ERROR_ACCESSING_SW360)
+
         sw360id = CycloneDxSupport.get_property_value(component, CycloneDxSupport.CDX_PROP_SW360ID)
+        version = component.version or ""
         for step in range(3):
             try:
                 release_details = self.client.get_release(sw360id)
                 return release_details
-            except sw360.sw360_api.SW360Error as swex:
-                if swex.response.status_code == requests.codes['not_found']:
+            except SW360Error as swex:
+                if swex.response is None:
+                    print_red("  Unknown error: " + swex.message)
+                elif swex.response.status_code == requests.codes['not_found']:
                     print_yellow(
                         "  Not found " + component.name +
-                        ", " + component.version + ", " + sw360id)
+                        ", " + version + ", " + sw360id)
                     break
 
                 # only report other errors if this is the third attempt
                 if step >= 2:
                     print(Fore.LIGHTRED_EX + "  Error retrieving release data: ")
                     print(
-                        "  " + component.name + ", " + component.version +
+                        "  " + component.name + ", " + version +
                         ", " + sw360id)
-                    print("  Status Code: " + str(swex.response.status_code))
+                    if swex.response:
+                        print("  Status Code: " + str(swex.response.status_code))
                     if swex.message:
                         print("    Message: " + swex.message)
                     print(Style.RESET_ALL)
 
         return None
 
-    def _find_by_name(self, component: Component) -> dict:
+    def _find_by_name(self, component: Component) -> Optional[Dict[str, Any]]:
+        if not self.client:
+            print_red("  No client!")
+            sys.exit(ResultCode.RESULT_ERROR_ACCESSING_SW360)
+
+        version = component.version or ""
         for step in range(3):
             try:
                 releases = self.client.get_releases_by_name(component.name)
                 if not releases:
                     return None
 
                 for r in releases:
-                    if r.get("version", "") == component.version:
+                    if r.get("version", "") == version:
                         return r
 
                 return None
-            except sw360.sw360_api.SW360Error as swex:
-                if swex.response.status_code == requests.codes['not_found']:
+            except SW360Error as swex:
+                if swex.response is None:
+                    print_red("  Unknown error: " + swex.message)
+                elif swex.response.status_code == requests.codes['not_found']:
                     print_yellow(
                         "  Not found " + component.name +
-                        ", " + component.version)
+                        ", " + version)
                     break
 
                 # only report other errors if this is the third attempt
                 if step >= 2:
                     print(Fore.LIGHTRED_EX + "  Error retrieving release data: ")
                     print(
-                        "  " + component.name + ", " + component.version)
-                    print("  Status Code: " + str(swex.response.status_code))
+                        "  " + component.name + ", " + version)
+                    if swex.response:
+                        print("  Status Code: " + str(swex.response.status_code))
                     if swex.message:
                         print("    Message: " + swex.message)
                     print(Style.RESET_ALL)
@@ -99,6 +116,10 @@ def _find_by_name(self, component: Component) -> dict:
     def check_releases(self, bom: Bom) -> int:
         """Checks for each release in the list whether it can be found on the specified
         SW360 instance."""
+        if not self.client:
+            print_red("  No client!")
+            sys.exit(ResultCode.RESULT_ERROR_ACCESSING_SW360)
+
         found_count = 0
         for component in bom.components:
             release_details = None
@@ -116,7 +137,7 @@ def check_releases(self, bom: Bom) -> int:
                 found_count += 1
                 continue
 
-            if not id:
+            if not sw360id:
                 print_yellow(
                     "  " + component.name +
                     ", " + component.version +
@@ -125,7 +146,7 @@ def check_releases(self, bom: Bom) -> int:
 
         return found_count
 
-    def run(self, args):
+    def run(self, args: Any) -> None:
         """Main method()"""
         if args.debug:
             global LOG

capycli/bom/check_bom_item_status.py

@@ -1,5 +1,5 @@
 # -------------------------------------------------------------------------------
-# Copyright (c) 2019-23 Siemens
+# Copyright (c) 2019-2024 Siemens
 # All Rights Reserved.
 # Author: thomas.graf@siemens.com
 #
@@ -12,10 +12,10 @@
 from typing import Any, Dict, Optional
 
 import requests
-import sw360.sw360_api
 from colorama import Fore, Style
 from cyclonedx.model.bom import Bom
 from cyclonedx.model.component import Component
+from sw360 import SW360Error
 
 import capycli.common.script_base
 from capycli.common.capycli_bom_support import CaPyCliBom, CycloneDxSupport
@@ -39,20 +39,26 @@ def _bom_has_items_without_id(self, bom: Bom) -> bool:
         return False
 
     def _find_by_id(self, component: Component) -> Optional[Dict[str, Any]]:
+        if not self.client:
+            print_red("  No client!")
+            sys.exit(ResultCode.RESULT_ERROR_ACCESSING_SW360)
+
         sw360id = CycloneDxSupport.get_property_value(component, CycloneDxSupport.CDX_PROP_SW360ID)
+        version = component.version or ""
         try:
             release_details = self.client.get_release(sw360id)
             return release_details
-        except sw360.sw360_api.SW360Error as swex:
-            if swex.response.status_code == requests.codes['not_found']:
+        except SW360Error as swex:
+            if swex.response is None:
+                print_red("  Unknown error: " + swex.message)
+            elif swex.response.status_code == requests.codes['not_found']:
                 print(
                     Fore.LIGHTYELLOW_EX + "  Not found " + component.name +
-                    ", " + component.version + ", " +
-                    sw360id + Style.RESET_ALL)
+                    ", " + version + ", " + sw360id + Style.RESET_ALL)
             else:
                 print(Fore.LIGHTRED_EX + "  Error retrieving release data: ")
                 print(
-                    "  " + str(component.name) + ", " + str(component.version) +
+                    "  " + component.name + ", " + version +
                     ", " + sw360id)
                 print("  Status Code: " + str(swex.response.status_code))
                 if swex.message:
@@ -62,25 +68,32 @@ def _find_by_id(self, component: Component) -> Optional[Dict[str, Any]]:
         return None
 
     def _find_by_name(self, component: Component) -> Optional[Dict[str, Any]]:
+        if not self.client:
+            print_red("  No client!")
+            sys.exit(ResultCode.RESULT_ERROR_ACCESSING_SW360)
+
+        version = component.version or ""
         try:
             releases = self.client.get_releases_by_name(component.name)
             if not releases:
                 return None
 
             for r in releases:
-                if r.get("version", "") == component.version:
+                if r.get("version", "") == version:
                     return self.client.get_release_by_url(r["_links"]["self"]["href"])
 
             return None
-        except sw360.sw360_api.SW360Error as swex:
-            if swex.response.status_code == requests.codes['not_found']:
+        except SW360Error as swex:
+            if swex.response is None:
+                print_red("  Unknown error: " + swex.message)
+            elif swex.response.status_code == requests.codes['not_found']:
                 print(
                     Fore.LIGHTYELLOW_EX + "  Not found " + component.name +
-                    ", " + component.version + ", " +
+                    ", " + version + ", " +
                     Style.RESET_ALL)
             else:
                 print(Fore.LIGHTRED_EX + "  Error retrieving release data: ")
-                print("  " + str(component.name) + ", " + str(component.version))
+                print("  " + str(component.name) + ", " + str(version))
                 print("  Status Code: " + str(swex.response.status_code))
                 if swex.message:
                     print("    Message: " + swex.message)
@@ -89,6 +102,10 @@ def _find_by_name(self, component: Component) -> Optional[Dict[str, Any]]:
             return None
 
     def show_bom_item_status(self, bom: Bom, all: bool = False) -> None:
+        if not self.client:
+            print_red("  No client!")
+            sys.exit(ResultCode.RESULT_ERROR_ACCESSING_SW360)
+
         for component in bom.components:
             release = None
             id = CycloneDxSupport.get_property_value(component, CycloneDxSupport.CDX_PROP_SW360ID)
@@ -117,13 +134,19 @@ def show_bom_item_status(self, bom: Bom, all: bool = False) -> None:
                         release["_links"]["sw360:component"]["href"]
                     )
                 )
+                if not comp_sw360:
+                    print_red("Error accessing component")
+                    continue
 
                 rel_list = comp_sw360["_embedded"]["sw360:releases"]
                 print("  " + component.name + ", " + component.version + " => ", end="", flush=True)
                 print("releases for component found = " + str(len(rel_list)))
                 for orel in rel_list:
                     href = orel["_links"]["self"]["href"]
                     rel = self.client.get_release_by_url(href)
+                    if not rel:
+                        print_red("Error accessing release " + href)
+                        continue
                     cs = rel.get("clearingState", "(unkown clearing state)")
                     if cs == "APPROVED":
                         print(Fore.LIGHTGREEN_EX, end="", flush=True)
@@ -141,7 +164,7 @@ def show_bom_item_status(self, bom: Bom, all: bool = False) -> None:
                     " => --- no id ---")
                 continue
 
-    def run(self, args) -> None:
+    def run(self, args: Any) -> None:
         """Main method()"""
         if args.debug:
             global LOG