Feature/595 create vulnerability tracking class (#597)

* Group poetry.lock and pyproject.toml in frozen dataclass

* Move PoetryFiles and create poetry_files_from_latest_tag in shared_models. Add get_vulnerabilities_from_latest_tag

* Add test for poetry_files_from_latest_tag

* Add basic unit tests for get_vulnerabilities and get_vulnerabilities_from_latest_tag

* Add ResolvedVulnerabilities to detect if vulnerability has been
resolved from previous work to the present

* Add coordinates

* Move VulnerabilitySource to audit.py for shared usage

* Move references_links to Vulnerability

* Add vulnerability_id and subsection_for_changelog_summary with tests

* Make docstring sentence clearer what the intent is by matching ID and aliases per review

* Change structure of vulnerability so clearer delineation between package information and vulnerability

Author: ArBridgeman

doc/changes/unreleased.md

@@ -6,6 +6,10 @@
 * #535: Added more information about Sonar's usage of ``exclusions``
 * #596: Corrected and added more information regarding ``pyupgrade``
 
+## Features
+
+* #595: Created class `ResolvedVulnerabilities` to track resolved vulnerabilities between versions
+
 ## Refactoring
 
 * #596: Added newline after header in versioned changelog

exasol/toolbox/nox/_lint.py

@@ -11,6 +11,7 @@
 from nox import Session
 
 from exasol.toolbox.nox._shared import python_files
+from exasol.toolbox.util.dependencies.shared_models import PoetryFiles
 from noxconfig import PROJECT_CONFIG
 
 
@@ -140,7 +141,7 @@ def security_lint(session: Session) -> None:
 @nox.session(name="lint:dependencies", python=False)
 def dependency_check(session: Session) -> None:
     """Checks if only valid sources of dependencies are used"""
-    content = Path(PROJECT_CONFIG.root, "pyproject.toml").read_text()
+    content = Path(PROJECT_CONFIG.root, PoetryFiles.pyproject_toml).read_text()
     dependencies = Dependencies.parse(content)
     console = rich.console.Console()
     if illegal := dependencies.illegal:

exasol/toolbox/nox/_release.py

@@ -14,6 +14,7 @@
     check_for_config_attribute,
 )
 from exasol.toolbox.nox.plugin import NoxTasks
+from exasol.toolbox.util.dependencies.shared_models import PoetryFiles
 from exasol.toolbox.util.git import Git
 from exasol.toolbox.util.release.changelog import Changelogs
 from exasol.toolbox.util.version import (
@@ -142,7 +143,7 @@ def prepare_release(session: Session) -> None:
         return
 
     changed_files += [
-        PROJECT_CONFIG.root / "pyproject.toml",
+        PROJECT_CONFIG.root / PoetryFiles.pyproject_toml,
         PROJECT_CONFIG.version_file,
     ]
     results = pm.hook.prepare_release_add_files(session=session, config=PROJECT_CONFIG)

exasol/toolbox/tools/security.py

@@ -18,10 +18,11 @@
 from functools import partial
 from inspect import cleandoc
 from pathlib import Path
-from typing import Optional
 
 import typer
 
+from exasol.toolbox.util.dependencies.audit import VulnerabilitySource
+
 stdout = print
 stderr = partial(print, file=sys.stderr)
 
@@ -104,45 +105,14 @@ def from_maven(report: str) -> Iterable[Issue]:
             )
 
 
-class VulnerabilitySource(str, Enum):
-    CVE = "CVE"
-    CWE = "CWE"
-    GHSA = "GHSA"
-    PYSEC = "PYSEC"
-
-    @classmethod
-    def from_prefix(cls, name: str) -> VulnerabilitySource | None:
-        for el in cls:
-            if name.upper().startswith(el.value):
-                return el
-        return None
-
-    def get_link(self, package: str, vuln_id: str) -> str:
-        if self == VulnerabilitySource.CWE:
-            cwe_id = vuln_id.upper().replace(f"{VulnerabilitySource.CWE.value}-", "")
-            return f"https://cwe.mitre.org/data/definitions/{cwe_id}.html"
-
-        map_link = {
-            VulnerabilitySource.CVE: "https://nvd.nist.gov/vuln/detail/{vuln_id}",
-            VulnerabilitySource.GHSA: "https://github.com/advisories/{vuln_id}",
-            VulnerabilitySource.PYSEC: "https://github.com/pypa/advisory-database/blob/main/vulns/{package}/{vuln_id}.yaml",
-        }
-        return map_link[self].format(package=package, vuln_id=vuln_id)
-
-
-def identify_pypi_references(
-    references: list[str], package_name: str
-) -> tuple[list[str], list[str], list[str]]:
+def identify_pypi_references(references: list[str]) -> tuple[list[str], list[str]]:
     refs: dict = {k: [] for k in VulnerabilitySource}
-    links = []
     for reference in references:
         if source := VulnerabilitySource.from_prefix(reference.upper()):
             refs[source].append(reference)
-            links.append(source.get_link(package=package_name, vuln_id=reference))
     return (
         refs[VulnerabilitySource.CVE],
         refs[VulnerabilitySource.CWE],
-        links,
     )
 
 
@@ -167,6 +137,11 @@ def from_pip_audit(report: str) -> Iterable[Issue]:
               "CVE-2025-27516"
             ],
             "description": "An oversight ..."
+            "coordinates": "jinja2:3.1.5",
+            "references": [
+              "https://github.com/advisories/GHSA-cpwx-vrp4-4pq7",
+              "https://nvd.nist.gov/vuln/detail/CVE-2025-27516"
+            ]
           }
         ]
 
@@ -178,16 +153,16 @@ def from_pip_audit(report: str) -> Iterable[Issue]:
     vulnerabilities = json.loads(report)
 
     for vulnerability in vulnerabilities:
-        cves, cwes, links = identify_pypi_references(
-            references=vulnerability["refs"], package_name=vulnerability["name"]
+        cves, cwes = identify_pypi_references(
+            references=vulnerability["refs"],
         )
         if cves:
             yield Issue(
                 cve=sorted(cves)[0],
                 cwe="None" if not cwes else ", ".join(cwes),
                 description=vulnerability["description"],
-                coordinates=f"{vulnerability['name']}:{vulnerability['version']}",
-                references=tuple(links),
+                coordinates=vulnerability["coordinates"],
+                references=tuple(vulnerability["references"]),
             )
 
 

exasol/toolbox/util/dependencies/audit.py

@@ -4,16 +4,23 @@
 import subprocess  # nosec
 import tempfile
 from dataclasses import dataclass
+from enum import Enum
+from inspect import cleandoc
 from pathlib import Path
 from re import search
 from typing import (
     Any,
-    Union,
 )
 
-from pydantic import BaseModel
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+)
 
-from exasol.toolbox.util.dependencies.shared_models import Package
+from exasol.toolbox.util.dependencies.shared_models import (
+    Package,
+    poetry_files_from_latest_tag,
+)
 
 PIP_AUDIT_VULNERABILITY_PATTERN = (
     r"^Found \d+ known vulnerabilit\w{1,3} in \d+ package\w?$"
@@ -32,7 +39,36 @@ def __init__(self, subprocess_output: subprocess.CompletedProcess) -> None:
         self.stderr = subprocess_output.stderr
 
 
-class Vulnerability(Package):
+class VulnerabilitySource(str, Enum):
+    CVE = "CVE"
+    CWE = "CWE"
+    GHSA = "GHSA"
+    PYSEC = "PYSEC"
+
+    @classmethod
+    def from_prefix(cls, name: str) -> VulnerabilitySource | None:
+        for el in cls:
+            if name.upper().startswith(el.value):
+                return el
+        return None
+
+    def get_link(self, package: str, vuln_id: str) -> str:
+        if self == VulnerabilitySource.CWE:
+            cwe_id = vuln_id.upper().replace(f"{VulnerabilitySource.CWE.value}-", "")
+            return f"https://cwe.mitre.org/data/definitions/{cwe_id}.html"
+
+        map_link = {
+            VulnerabilitySource.CVE: "https://nvd.nist.gov/vuln/detail/{vuln_id}",
+            VulnerabilitySource.GHSA: "https://github.com/advisories/{vuln_id}",
+            VulnerabilitySource.PYSEC: "https://github.com/pypa/advisory-database/blob/main/vulns/{package}/{vuln_id}.yaml",
+        }
+        return map_link[self].format(package=package, vuln_id=vuln_id)
+
+
+class Vulnerability(BaseModel):
+    model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True)
+
+    package: Package
     id: str
     aliases: list[str]
     fix_versions: list[str]
@@ -46,23 +82,61 @@ def from_audit_entry(
         Create a Vulnerability from a pip-audit vulnerability entry
         """
         return cls(
-            name=package_name,
-            version=version,
+            package=Package(name=package_name, version=version),
             id=vuln_entry["id"],
             aliases=vuln_entry["aliases"],
             fix_versions=vuln_entry["fix_versions"],
             description=vuln_entry["description"],
         )
 
     @property
-    def security_issue_entry(self) -> dict[str, str | list[str]]:
+    def references(self) -> list[str]:
+        return sorted([self.id] + self.aliases)
+
+    @property
+    def reference_links(self) -> tuple[str, ...]:
+        return tuple(
+            source.get_link(package=self.package.name, vuln_id=reference)
+            for reference in self.references
+            if (source := VulnerabilitySource.from_prefix(reference.upper()))
+        )
+
+    @property
+    def security_issue_entry(self) -> dict[str, str | list[str] | tuple[str, ...]]:
         return {
-            "name": self.name,
-            "version": str(self.version),
-            "refs": [self.id] + self.aliases,
+            "name": self.package.name,
+            "version": str(self.package.version),
+            "refs": self.references,
             "description": self.description,
+            "coordinates": self.package.coordinates,
+            "references": self.reference_links,
         }
 
+    @property
+    def vulnerability_id(self) -> str | None:
+        """
+        Ensure a consistent way of identifying a vulnerability for string generation.
+        """
+        for ref in self.references:
+            ref_upper = ref.upper()
+            if ref_upper.startswith(VulnerabilitySource.CVE.value):
+                return ref
+            if ref_upper.startswith(VulnerabilitySource.GHSA.value):
+                return ref
+            if ref_upper.startswith(VulnerabilitySource.PYSEC.value):
+                return ref
+        return self.references[0]
+
+    @property
+    def subsection_for_changelog_summary(self) -> str:
+        """
+        Create a subsection to be included in the Summary section of a versioned changelog.
+        """
+        links_join = "\n* ".join(sorted(self.reference_links))
+        references_subsection = f"\n#### References:\n\n* {links_join}\n\n "
+        subsection = f"### {self.vulnerability_id} in {self.package.coordinates}\n\n{self.description}\n{references_subsection}"
+        return cleandoc(subsection.strip())
+
 
 def audit_poetry_files(working_directory: Path) -> str:
     """
@@ -141,7 +215,18 @@ def load_from_pip_audit(cls, working_directory: Path) -> Vulnerabilities:
         return Vulnerabilities(vulnerabilities=vulnerabilities)
 
     @property
-    def security_issue_dict(self) -> list[dict[str, str | list[str]]]:
+    def security_issue_dict(self) -> list[dict[str, str | list[str] | tuple[str, ...]]]:
         return [
             vulnerability.security_issue_entry for vulnerability in self.vulnerabilities
         ]
+
+
+def get_vulnerabilities(working_directory: Path) -> list[Vulnerability]:
+    return Vulnerabilities.load_from_pip_audit(
+        working_directory=working_directory
+    ).vulnerabilities
+
+
+def get_vulnerabilities_from_latest_tag():
+    with poetry_files_from_latest_tag() as tmp_dir:
+        return get_vulnerabilities(working_directory=tmp_dir)

exasol/toolbox/util/dependencies/poetry_dependencies.py

@@ -1,10 +1,8 @@
 from __future__ import annotations
 
 import subprocess
-import tempfile
 from collections import OrderedDict
 from pathlib import Path
-from typing import Optional
 
 import tomlkit
 from pydantic import (
@@ -16,9 +14,9 @@
 from exasol.toolbox.util.dependencies.shared_models import (
     NormalizedPackageStr,
     Package,
+    PoetryFiles,
+    poetry_files_from_latest_tag,
 )
-from exasol.toolbox.util.git import Git
-from noxconfig import PROJECT_CONFIG
 
 
 class PoetryGroup(BaseModel):
@@ -28,7 +26,6 @@ class PoetryGroup(BaseModel):
     toml_section: str | None
 
 
-PYPROJECT_TOML = "pyproject.toml"
 TRANSITIVE_GROUP = PoetryGroup(name="transitive", toml_section=None)
 
 
@@ -39,7 +36,7 @@ class PoetryToml(BaseModel):
 
     @classmethod
     def load_from_toml(cls, working_directory: Path) -> PoetryToml:
-        file_path = working_directory / PYPROJECT_TOML
+        file_path = working_directory / PoetryFiles.pyproject_toml
         if not file_path.exists():
             raise ValueError(f"File not found: {file_path}")
 
@@ -165,10 +162,5 @@ def get_dependencies(
 def get_dependencies_from_latest_tag() -> (
     OrderedDict[str, dict[NormalizedPackageStr, Package]]
 ):
-    latest_tag = Git.get_latest_tag()
-    path = PROJECT_CONFIG.root.relative_to(Git.toplevel())
-    with tempfile.TemporaryDirectory() as tmpdir_str:
-        tmpdir = Path(tmpdir_str)
-        for file in ("poetry.lock", PYPROJECT_TOML):
-            Git.checkout(latest_tag, path / file, tmpdir / file)
-        return get_dependencies(working_directory=tmpdir)
+    with poetry_files_from_latest_tag() as tmp_dir:
+        return get_dependencies(working_directory=tmp_dir)