fix(python): add poetry v2 support [backport: release/v0.59] (#8335)

Co-authored-by: DmitriyLewen <91113035+DmitriyLewen@users.noreply.github.com>
Co-authored-by: Nikita Pivkin <nikita.pivkin@smartforce.io>

Author: 3 people

pkg/dependency/parser/python/poetry/parse.go

@@ -1,6 +1,7 @@
 package poetry
 
 import (
+	"slices"
 	"sort"
 
 	"github.com/BurntSushi/toml"
@@ -17,6 +18,7 @@ import (
 type Lockfile struct {
 	Packages []struct {
 		Category       string         `toml:"category"`
+		Groups         []string       `toml:"groups"`
 		Description    string         `toml:"description"`
 		Marker         string         `toml:"marker,omitempty"`
 		Name           string         `toml:"name"`
@@ -50,15 +52,16 @@ func (p *Parser) Parse(r xio.ReadSeekerAt) ([]ftypes.Package, []ftypes.Dependenc
 	var pkgs []ftypes.Package
 	var deps []ftypes.Dependency
 	for _, pkg := range lockfile.Packages {
-		if pkg.Category == "dev" {
-			continue
-		}
-
 		pkgID := packageID(pkg.Name, pkg.Version)
 		pkgs = append(pkgs, ftypes.Package{
 			ID:      pkgID,
 			Name:    pkg.Name,
 			Version: pkg.Version,
+			// TODO upgrade logic for working with groups
+			// Mark only:
+			// - `category = "dev"`
+			// - groups without `main`. e.g. `groups = ["dev"]`
+			Dev: pkg.Category == "dev" || (len(pkg.Groups) > 0 && !slices.Contains(pkg.Groups, "main")),
 		})
 
 		dependsOn := p.parseDependencies(pkg.Dependencies, pkgVersions)

pkg/dependency/parser/python/poetry/parse_test.go

@@ -25,20 +25,20 @@ func TestParser_Parse(t *testing.T) {
 			wantPkgs: poetryNormal,
 			wantErr:  assert.NoError,
 		},
-		{
-			name:     "many",
-			file:     "testdata/poetry_many.lock",
-			wantPkgs: poetryMany,
-			wantDeps: poetryManyDeps,
-			wantErr:  assert.NoError,
-		},
 		{
 			name:     "flask",
 			file:     "testdata/poetry_flask.lock",
 			wantPkgs: poetryFlask,
 			wantDeps: poetryFlaskDeps,
 			wantErr:  assert.NoError,
 		},
+		{
+			name:     "flask + poetry v2",
+			file:     "testdata/poetry_v2_flask.lock",
+			wantPkgs: poetryFlask,
+			wantDeps: poetryV2FlaskDeps,
+			wantErr:  assert.NoError,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

pkg/dependency/parser/python/poetry/parse_testcase.go

@@ -14,122 +14,55 @@ var (
 		{ID: "pypi@2.1", Name: "pypi", Version: "2.1"},
 	}
 
-	// docker run --name poetry --rm -it python@sha256:e1141f10176d74d1a0e87a7c0a0a5a98dd98ec5ac12ce867768f40c6feae2fd9 sh
-	// apk add curl
-	// curl -sSL https://install.python-poetry.org | POETRY_VERSION=1.1.7 python3 -
-	// export PATH=/root/.local/bin:$PATH
-	// poetry new many && cd many
-	// curl -o poetry.lock https://raw.githubusercontent.com/python-poetry/poetry/c8945eb110aeda611cc6721565d7ad0c657d453a/poetry.lock
-	// curl -o pyproject.toml https://raw.githubusercontent.com/python-poetry/poetry/c8945eb110aeda611cc6721565d7ad0c657d453a/pyproject.toml
-	// poetry show -a | awk '{gsub(/\(!\)/, ""); printf("{ID: \""$1"@"$2"\", Name: \""$1"\", Version: \""$2"\"},\n") }'
-	// `--no-dev` flag uncorrected returns deps. Then need to remove `dev` deps manually
-	// list of dev deps - cat poetry.lock | grep 'category = "dev"' -B 3
-	poetryMany = []ftypes.Package{
-		{ID: "attrs@22.2.0", Name: "attrs", Version: "22.2.0"},
-		{ID: "backports-cached-property@1.0.2", Name: "backports-cached-property", Version: "1.0.2"},
-		{ID: "build@0.10.0", Name: "build", Version: "0.10.0"},
-		{ID: "cachecontrol@0.12.11", Name: "cachecontrol", Version: "0.12.11"},
-		{ID: "certifi@2022.12.7", Name: "certifi", Version: "2022.12.7"},
-		{ID: "cffi@1.15.1", Name: "cffi", Version: "1.15.1"},
-		{ID: "charset-normalizer@3.0.1", Name: "charset-normalizer", Version: "3.0.1"},
-		{ID: "cleo@2.0.1", Name: "cleo", Version: "2.0.1"},
-		{ID: "colorama@0.4.6", Name: "colorama", Version: "0.4.6"},
-		{ID: "crashtest@0.4.1", Name: "crashtest", Version: "0.4.1"},
-		{ID: "cryptography@39.0.0", Name: "cryptography", Version: "39.0.0"},
-		{ID: "distlib@0.3.6", Name: "distlib", Version: "0.3.6"},
-		{ID: "dulwich@0.21.2", Name: "dulwich", Version: "0.21.2"},
-		{ID: "filelock@3.9.0", Name: "filelock", Version: "3.9.0"},
-		{ID: "html5lib@1.1", Name: "html5lib", Version: "1.1"},
-		{ID: "idna@3.4", Name: "idna", Version: "3.4"},
-		{ID: "importlib-metadata@6.0.0", Name: "importlib-metadata", Version: "6.0.0"},
-		{ID: "importlib-resources@5.10.2", Name: "importlib-resources", Version: "5.10.2"},
-		{ID: "installer@0.6.0", Name: "installer", Version: "0.6.0"},
-		{ID: "jaraco-classes@3.2.3", Name: "jaraco-classes", Version: "3.2.3"},
-		{ID: "jeepney@0.8.0", Name: "jeepney", Version: "0.8.0"},
-		{ID: "jsonschema@4.17.3", Name: "jsonschema", Version: "4.17.3"},
-		{ID: "keyring@23.13.1", Name: "keyring", Version: "23.13.1"},
-		{ID: "lockfile@0.12.2", Name: "lockfile", Version: "0.12.2"},
-		{ID: "more-itertools@9.0.0", Name: "more-itertools", Version: "9.0.0"},
-		{ID: "msgpack@1.0.4", Name: "msgpack", Version: "1.0.4"},
-		{ID: "packaging@23.0", Name: "packaging", Version: "23.0"},
-		{ID: "pexpect@4.8.0", Name: "pexpect", Version: "4.8.0"},
-		{ID: "pkginfo@1.9.6", Name: "pkginfo", Version: "1.9.6"},
-		{ID: "pkgutil-resolve-name@1.3.10", Name: "pkgutil-resolve-name", Version: "1.3.10"},
-		{ID: "platformdirs@2.6.2", Name: "platformdirs", Version: "2.6.2"},
-		{ID: "poetry-core@1.5.0", Name: "poetry-core", Version: "1.5.0"},
-		{ID: "poetry-plugin-export@1.3.0", Name: "poetry-plugin-export", Version: "1.3.0"},
-		{ID: "ptyprocess@0.7.0", Name: "ptyprocess", Version: "0.7.0"},
-		{ID: "pycparser@2.21", Name: "pycparser", Version: "2.21"},
-		{ID: "pyproject-hooks@1.0.0", Name: "pyproject-hooks", Version: "1.0.0"},
-		{ID: "pyrsistent@0.19.3", Name: "pyrsistent", Version: "0.19.3"},
-		{ID: "pywin32-ctypes@0.2.0", Name: "pywin32-ctypes", Version: "0.2.0"},
-		{ID: "rapidfuzz@2.13.7", Name: "rapidfuzz", Version: "2.13.7"},
-		{ID: "requests@2.28.2", Name: "requests", Version: "2.28.2"},
-		{ID: "requests-toolbelt@0.10.1", Name: "requests-toolbelt", Version: "0.10.1"},
-		{ID: "secretstorage@3.3.3", Name: "secretstorage", Version: "3.3.3"},
-		{ID: "shellingham@1.5.0.post1", Name: "shellingham", Version: "1.5.0.post1"},
-		{ID: "six@1.16.0", Name: "six", Version: "1.16.0"},
-		{ID: "tomli@2.0.1", Name: "tomli", Version: "2.0.1"},
-		{ID: "tomlkit@0.11.6", Name: "tomlkit", Version: "0.11.6"},
-		{ID: "trove-classifiers@2023.1.20", Name: "trove-classifiers", Version: "2023.1.20"},
-		{ID: "typing-extensions@4.4.0", Name: "typing-extensions", Version: "4.4.0"},
-		{ID: "urllib3@1.26.14", Name: "urllib3", Version: "1.26.14"},
-		{ID: "virtualenv@20.16.5", Name: "virtualenv", Version: "20.16.5"},
-		{ID: "virtualenv@20.17.1", Name: "virtualenv", Version: "20.17.1"},
-		{ID: "webencodings@0.5.1", Name: "webencodings", Version: "0.5.1"},
-		{ID: "xattr@0.10.1", Name: "xattr", Version: "0.10.1"},
-		{ID: "zipp@3.12.0", Name: "zipp", Version: "3.12.0"},
-	}
-
-	// cat poetry.lock | grep "\[package.dependencies\]" -B 3 -A 8 - it might help to complete this slice
-	poetryManyDeps = []ftypes.Dependency{
-		{ID: "build@0.10.0", DependsOn: []string{"colorama@0.4.6", "importlib-metadata@6.0.0", "packaging@23.0", "pyproject-hooks@1.0.0", "tomli@2.0.1"}},
-		{ID: "cachecontrol@0.12.11", DependsOn: []string{"lockfile@0.12.2", "msgpack@1.0.4", "requests@2.28.2"}},
-		{ID: "cffi@1.15.1", DependsOn: []string{"pycparser@2.21"}},
-		{ID: "cleo@2.0.1", DependsOn: []string{"crashtest@0.4.1", "rapidfuzz@2.13.7"}},
-		{ID: "cryptography@39.0.0", DependsOn: []string{"cffi@1.15.1"}},
-		{ID: "dulwich@0.21.2", DependsOn: []string{"typing-extensions@4.4.0", "urllib3@1.26.14"}},
-		{ID: "html5lib@1.1", DependsOn: []string{"six@1.16.0", "webencodings@0.5.1"}},
-		{ID: "importlib-metadata@6.0.0", DependsOn: []string{"typing-extensions@4.4.0", "zipp@3.12.0"}},
-		{ID: "importlib-resources@5.10.2", DependsOn: []string{"zipp@3.12.0"}},
-		{ID: "jaraco-classes@3.2.3", DependsOn: []string{"more-itertools@9.0.0"}},
-		{ID: "jsonschema@4.17.3", DependsOn: []string{"attrs@22.2.0", "importlib-metadata@6.0.0", "importlib-resources@5.10.2", "pkgutil-resolve-name@1.3.10", "pyrsistent@0.19.3", "typing-extensions@4.4.0"}},
-		{ID: "keyring@23.13.1", DependsOn: []string{"importlib-metadata@6.0.0", "importlib-resources@5.10.2", "jaraco-classes@3.2.3", "jeepney@0.8.0", "pywin32-ctypes@0.2.0", "secretstorage@3.3.3"}},
-		{ID: "pexpect@4.8.0", DependsOn: []string{"ptyprocess@0.7.0"}},
-		{ID: "platformdirs@2.6.2", DependsOn: []string{"typing-extensions@4.4.0"}},
-		{ID: "poetry-core@1.5.0", DependsOn: []string{"importlib-metadata@6.0.0"}},
-		{ID: "poetry-plugin-export@1.3.0", DependsOn: []string{"poetry-core@1.5.0"}},
-		{ID: "pyproject-hooks@1.0.0", DependsOn: []string{"tomli@2.0.1"}},
-		{ID: "requests@2.28.2", DependsOn: []string{"certifi@2022.12.7", "charset-normalizer@3.0.1", "idna@3.4", "urllib3@1.26.14"}},
-		{ID: "requests-toolbelt@0.10.1", DependsOn: []string{"requests@2.28.2"}},
-		{ID: "secretstorage@3.3.3", DependsOn: []string{"cryptography@39.0.0", "jeepney@0.8.0"}},
-		{ID: "virtualenv@20.16.5", DependsOn: []string{"distlib@0.3.6", "filelock@3.9.0", "platformdirs@2.6.2"}},
-		{ID: "virtualenv@20.17.1", DependsOn: []string{"distlib@0.3.6", "filelock@3.9.0", "importlib-metadata@6.0.0", "platformdirs@2.6.2"}},
-		{ID: "xattr@0.10.1", DependsOn: []string{"cffi@1.15.1"}},
-	}
-
 	// docker run --name poetry --rm -it python@sha256:e1141f10176d74d1a0e87a7c0a0a5a98dd98ec5ac12ce867768f40c6feae2fd9 sh
 	// apk add curl
 	// curl -sSL https://install.python-poetry.org | POETRY_VERSION=1.1.7 python3 -
 	// export PATH=/root/.local/bin:$PATH
 	// poetry new web && cd web
+	// poetry add pluggy@0.13.1 -E pre-commit,tox
 	// poetry add flask@1.0.3
+	// poetry add pytest@5.4.3 --dev
 	// poetry show -a | awk '{gsub(/\(!\)/, ""); printf("{ID: \""$1"@"$2"\", Name: \""$1"\", Version: \""$2"\"},\n") }'
+	// mark dev deps
 	poetryFlask = []ftypes.Package{
-		{ID: "click@8.1.3", Name: "click", Version: "8.1.3"},
+		{ID: "atomicwrites@1.4.1", Name: "atomicwrites", Version: "1.4.1", Dev: true},
+		{ID: "attrs@25.1.0", Name: "attrs", Version: "25.1.0", Dev: true},
+		{ID: "click@8.1.8", Name: "click", Version: "8.1.8"},
 		{ID: "colorama@0.4.6", Name: "colorama", Version: "0.4.6"},
 		{ID: "flask@1.0.3", Name: "flask", Version: "1.0.3"},
-		{ID: "itsdangerous@2.1.2", Name: "itsdangerous", Version: "2.1.2"},
-		{ID: "jinja2@3.1.2", Name: "jinja2", Version: "3.1.2"},
-		{ID: "markupsafe@2.1.2", Name: "markupsafe", Version: "2.1.2"},
-		{ID: "werkzeug@2.2.3", Name: "werkzeug", Version: "2.2.3"},
+		{ID: "itsdangerous@2.2.0", Name: "itsdangerous", Version: "2.2.0"},
+		{ID: "jinja2@3.1.5", Name: "jinja2", Version: "3.1.5"},
+		{ID: "markupsafe@3.0.2", Name: "markupsafe", Version: "3.0.2"},
+		{ID: "more-itertools@10.6.0", Name: "more-itertools", Version: "10.6.0", Dev: true},
+		{ID: "packaging@24.2", Name: "packaging", Version: "24.2", Dev: true},
+		{ID: "pluggy@0.13.1", Name: "pluggy", Version: "0.13.1", Dev: false},
+		{ID: "py@1.11.0", Name: "py", Version: "1.11.0", Dev: true},
+		{ID: "pytest@5.4.3", Name: "pytest", Version: "5.4.3", Dev: true},
+		{ID: "wcwidth@0.2.13", Name: "wcwidth", Version: "0.2.13", Dev: true},
+		{ID: "werkzeug@3.1.3", Name: "werkzeug", Version: "3.1.3"},
 	}
 
 	// cat poetry.lock | grep "\[package.dependencies\]" -B 3 -A 8 - it might help to complete this slice
 	poetryFlaskDeps = []ftypes.Dependency{
-		{ID: "click@8.1.3", DependsOn: []string{"colorama@0.4.6"}},
-		{ID: "flask@1.0.3", DependsOn: []string{"click@8.1.3", "itsdangerous@2.1.2", "jinja2@3.1.2", "werkzeug@2.2.3"}},
-		{ID: "jinja2@3.1.2", DependsOn: []string{"markupsafe@2.1.2"}},
-		{ID: "werkzeug@2.2.3", DependsOn: []string{"markupsafe@2.1.2"}},
+		{ID: "click@8.1.8", DependsOn: []string{"colorama@0.4.6"}},
+		{ID: "flask@1.0.3", DependsOn: []string{"click@8.1.8", "itsdangerous@2.2.0", "jinja2@3.1.5", "werkzeug@3.1.3"}},
+		{ID: "jinja2@3.1.5", DependsOn: []string{"markupsafe@3.0.2"}},
+		{ID: "pytest@5.4.3", DependsOn: []string{"colorama@0.4.6", "pluggy@0.13.1"}},
+		{ID: "werkzeug@3.1.3", DependsOn: []string{"markupsafe@3.0.2"}},
+	}
+
+	// use instruction above with `POETRY_VERSION=2.0.1`
+
+	poetryV2FlaskDeps = []ftypes.Dependency{
+		{ID: "click@8.1.8", DependsOn: []string{"colorama@0.4.6"}},
+		{ID: "flask@1.0.3", DependsOn: []string{"click@8.1.8", "itsdangerous@2.2.0", "jinja2@3.1.5", "werkzeug@3.1.3"}},
+		{ID: "jinja2@3.1.5", DependsOn: []string{"markupsafe@3.0.2"}},
+		{
+			ID: "pytest@5.4.3", DependsOn: []string{
+				"atomicwrites@1.4.1", "attrs@25.1.0", "colorama@0.4.6", "more-itertools@10.6.0", "packaging@24.2",
+				"pluggy@0.13.1", "py@1.11.0", "wcwidth@0.2.13",
+			},
+		},
+		{ID: "werkzeug@3.1.3", DependsOn: []string{"markupsafe@3.0.2"}},
 	}
 )