Skip to content

Commit

Permalink
Merge pull request #8189 from dependabot/deivid-rodriguez/fix-multipl…
Browse files Browse the repository at this point in the history 
…e-pip-compile-errors

Fix multiple pip compile errors
  • Loading branch information
@deivid-rodriguez
deivid-rodriguez authored Oct 16, 2023
2 parents 5219691 + 450aa61 commit 2101a2a
Show file tree
Hide file tree
Showing 4 changed files with 139 additions and 95 deletions.
24 changes: 19 additions & 5 deletions python/lib/dependabot/python/file_parser.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,28 +74,42 @@ def requirement_dependencies
# probably blocked. Ignore it.
next if blocking_marker?(dep)

name = dep["name"]
file = dep["file"]
version = dep["version"]

requirements =
if lockfile_for_pip_compile_file?(dep["file"]) then []
if lockfile_for_pip_compile_file?(file) then []
else
[{
requirement: dep["requirement"],
file: Pathname.new(dep["file"]).cleanpath.to_path,
file: Pathname.new(file).cleanpath.to_path,
source: nil,
groups: group_from_filename(dep["file"])
groups: group_from_filename(file)
}]
end

# PyYAML < 6.0 will cause `pip-compile` to fail due to incompatiblity with Cython 3. Workaround it.
SharedHelpers.run_shell_command("pyenv exec pip install cython<3.0") if old_pyyaml?(name, version)

dependencies <<
Dependency.new(
name: normalised_name(dep["name"], dep["extras"]),
version: dep["version"]&.include?("*") ? nil : dep["version"],
name: normalised_name(name, dep["extras"]),
version: version&.include?("*") ? nil : version,
requirements: requirements,
package_manager: "pip"
)
end
dependencies
end

def old_pyyaml?(name, version)
major_version = version&.split(".")&.first
return false unless major_version

name == "pyyaml" && major_version < "6"
end

def group_from_filename(filename)
if filename.include?("dev") then ["dev-dependencies"]
else
Expand Down
66 changes: 43 additions & 23 deletions python/lib/dependabot/python/file_updater/pip_compile_file_updater.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,16 @@ class PipCompileFileUpdater
WARNINGS = /\s*# WARNING:.*\Z/m
UNSAFE_NOTE = /\s*# The following packages are considered to be unsafe.*\Z/m
RESOLVER_REGEX = /(?<=--resolver=)(\w+)/
NATIVE_COMPILATION_ERROR =
"pip._internal.exceptions.InstallationSubprocessError: Getting requirements to build wheel exited with 1"

attr_reader :dependencies, :dependency_files, :credentials

def initialize(dependencies:, dependency_files:, credentials:)
@dependencies = dependencies
@dependency_files = dependency_files
@credentials = credentials
@build_isolation = true
end

def updated_dependency_files
Expand Down Expand Up @@ -67,28 +70,7 @@ def compile_new_requirement_files
language_version_manager.install_required_python

filenames_to_compile.each do |filename|
# Shell out to pip-compile, generate a new set of requirements.
# This is slow, as pip-compile needs to do installs.
options = pip_compile_options(filename)
options_fingerprint = pip_compile_options_fingerprint(options)

name_part = "pyenv exec pip-compile " \
"#{options} -P " \
"#{dependency.name}"
fingerprint_name_part = "pyenv exec pip-compile " \
"#{options_fingerprint} -P " \
"<dependency_name>"

version_part = "#{dependency.version} #{filename}"
fingerprint_version_part = "<dependency_version> <filename>"

# Don't escape pyenv `dep-name==version` syntax
run_pip_compile_command(
"#{SharedHelpers.escape_command(name_part)}==" \
"#{SharedHelpers.escape_command(version_part)}",
allow_unsafe_shell_command: true,
fingerprint: "#{fingerprint_name_part}==#{fingerprint_version_part}"
)
compile_file(filename)
end

# Remove any .python-version file before parsing the reqs
Expand All @@ -108,6 +90,44 @@ def compile_new_requirement_files
end
end

def compile_file(filename)
# Shell out to pip-compile, generate a new set of requirements.
# This is slow, as pip-compile needs to do installs.
options = pip_compile_options(filename)
options_fingerprint = pip_compile_options_fingerprint(options)

name_part = "pyenv exec pip-compile " \
"#{options} -P " \
"#{dependency.name}"
fingerprint_name_part = "pyenv exec pip-compile " \
"#{options_fingerprint} -P " \
"<dependency_name>"

version_part = "#{dependency.version} #{filename}"
fingerprint_version_part = "<dependency_version> <filename>"

# Don't escape pyenv `dep-name==version` syntax
run_pip_compile_command(
"#{SharedHelpers.escape_command(name_part)}==" \
"#{SharedHelpers.escape_command(version_part)}",
allow_unsafe_shell_command: true,
fingerprint: "#{fingerprint_name_part}==#{fingerprint_version_part}"
)
rescue SharedHelpers::HelperSubprocessFailed => e
retry_count ||= 0
retry_count += 1
if compilation_error?(e) && retry_count <= 1
@build_isolation = false
retry
end

raise
end

def compilation_error?(error)
error.message.include?(NATIVE_COMPILATION_ERROR)
end

def update_manifest_files
dependency_files.filter_map do |file|
next unless file.name.end_with?(".in")
Expand Down Expand Up @@ -403,7 +423,7 @@ def pip_compile_options_fingerprint(options)
end

def pip_compile_options(filename)
options = ["--build-isolation"]
options = @build_isolation ? ["--build-isolation"] : ["--no-build-isolation"]
options += pip_compile_index_options

if (requirements_file = compiled_file_for_filename(filename))
Expand Down
136 changes: 71 additions & 65 deletions python/lib/dependabot/python/update_checker/pip_compile_version_resolver.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ class PipCompileVersionResolver
GIT_DEPENDENCY_UNREACHABLE_REGEX = /git clone --filter=blob:none --quiet (?<url>[^\s]+).* /
GIT_REFERENCE_NOT_FOUND_REGEX = /Did not find branch or tag '(?<tag>[^\n"]+)'/m
NATIVE_COMPILATION_ERROR =
"pip._internal.exceptions.InstallationSubprocessError: Command errored out with exit status 1:"
"pip._internal.exceptions.InstallationSubprocessError: Getting requirements to build wheel exited with 1"
# See https://packaging.python.org/en/latest/tutorials/packaging-projects/#configuring-metadata
PYTHON_PACKAGE_NAME_REGEX = /[A-Za-z0-9_\-]+/
RESOLUTION_IMPOSSIBLE_ERROR = "ResolutionImpossible"
Expand All @@ -43,17 +43,21 @@ def initialize(dependency:, dependency_files:, credentials:)
end

def latest_resolvable_version(requirement: nil)
@latest_resolvable_version_string ||= {}
return @latest_resolvable_version_string[requirement] if @latest_resolvable_version_string.key?(requirement)

version_string =
fetch_latest_resolvable_version_string(requirement: requirement)

version_string.nil? ? nil : Python::Version.new(version_string)
@latest_resolvable_version_string[requirement] ||=
version_string.nil? ? nil : Python::Version.new(version_string)
end

def resolvable?(version:)
@resolvable ||= {}
return @resolvable[version] if @resolvable.key?(version)

@resolvable[version] = if fetch_latest_resolvable_version_string(requirement: "==#{version}")
@resolvable[version] = if latest_resolvable_version(requirement: "==#{version}")
true
else
false
Expand All @@ -63,57 +67,59 @@ def resolvable?(version:)
private

def fetch_latest_resolvable_version_string(requirement:)
@latest_resolvable_version_string ||= {}
return @latest_resolvable_version_string[requirement] if @latest_resolvable_version_string.key?(requirement)
SharedHelpers.in_a_temporary_directory do
SharedHelpers.with_git_configured(credentials: credentials) do
write_temporary_dependency_files(updated_req: requirement)
language_version_manager.install_required_python

@latest_resolvable_version_string[requirement] ||=
SharedHelpers.in_a_temporary_directory do
SharedHelpers.with_git_configured(credentials: credentials) do
write_temporary_dependency_files(updated_req: requirement)
language_version_manager.install_required_python

filenames_to_compile.each do |filename|
# Shell out to pip-compile.
# This is slow, as pip-compile needs to do installs.
options = pip_compile_options(filename)
options_fingerprint = pip_compile_options_fingerprint(options)

run_pip_compile_command(
"pyenv exec pip-compile -v #{options} -P #{dependency.name} #{filename}",
fingerprint: "pyenv exec pip-compile -v #{options_fingerprint} -P <dependency_name> <filename>"
)

next if dependency.top_level?

# Run pip-compile a second time for transient dependencies
# to make sure we do not update dependencies that are
# superfluous. pip-compile does not detect these when
# updating a specific dependency with the -P option.
# Running pip-compile a second time will automatically remove
# superfluous dependencies. Dependabot then marks those with
# update_not_possible.
write_original_manifest_files
run_pip_compile_command(
"pyenv exec pip-compile #{options} #{filename}",
fingerprint: "pyenv exec pip-compile #{options_fingerprint} <filename>"
)
end

# Remove any .python-version file before parsing the reqs
FileUtils.remove_entry(".python-version", true)

parse_updated_files
end
rescue SharedHelpers::HelperSubprocessFailed => e
retry_count ||= 0
retry_count += 1
if compilation_error?(e) && retry_count <= 1
@build_isolation = false
retry
filenames_to_compile.each do |filename|
return nil unless compile_file(filename)
end

handle_pip_compile_errors(e)
# Remove any .python-version file before parsing the reqs
FileUtils.remove_entry(".python-version", true)

parse_updated_files
end
end
end

def compile_file(filename)
# Shell out to pip-compile.
# This is slow, as pip-compile needs to do installs.
options = pip_compile_options(filename)
options_fingerprint = pip_compile_options_fingerprint(options)

run_pip_compile_command(
"pyenv exec pip-compile -v #{options} -P #{dependency.name} #{filename}",
fingerprint: "pyenv exec pip-compile -v #{options_fingerprint} -P <dependency_name> <filename>"
)

return true if dependency.top_level?

# Run pip-compile a second time for transient dependencies
# to make sure we do not update dependencies that are
# superfluous. pip-compile does not detect these when
# updating a specific dependency with the -P option.
# Running pip-compile a second time will automatically remove
# superfluous dependencies. Dependabot then marks those with
# update_not_possible.
write_original_manifest_files
run_pip_compile_command(
"pyenv exec pip-compile #{options} #{filename}",
fingerprint: "pyenv exec pip-compile #{options_fingerprint} <filename>"
)

true
rescue SharedHelpers::HelperSubprocessFailed => e
retry_count ||= 0
retry_count += 1
if compilation_error?(e) && retry_count <= 1
@build_isolation = false
retry
end

handle_pip_compile_errors(e.message)
end

def compilation_error?(error)
Expand All @@ -122,23 +128,23 @@ def compilation_error?(error)

# rubocop:disable Metrics/AbcSize
# rubocop:disable Metrics/PerceivedComplexity
def handle_pip_compile_errors(error)
if error.message.include?(RESOLUTION_IMPOSSIBLE_ERROR)
def handle_pip_compile_errors(message)
if message.include?(RESOLUTION_IMPOSSIBLE_ERROR)
check_original_requirements_resolvable
# If the original requirements are resolvable but we get an
# incompatibility error after unlocking then it's likely to be
# due to problems with pip-compile's cascading resolution
return nil
end

if error.message.include?("UnsupportedConstraint")
if message.include?("UnsupportedConstraint")
# If there's an unsupported constraint, check if it existed
# previously (and raise if it did)
check_original_requirements_resolvable
end

if (error.message.include?('Command "python setup.py egg_info') ||
error.message.include?(
if (message.include?('Command "python setup.py egg_info') ||
message.include?(
"exit status 1: python setup.py egg_info"
)) &&
check_original_requirements_resolvable
Expand All @@ -147,16 +153,16 @@ def handle_pip_compile_errors(error)
return
end

if error.message.include?(RESOLUTION_IMPOSSIBLE_ERROR) &&
!error.message.match?(/#{Regexp.quote(dependency.name)}/i)
if message.include?(RESOLUTION_IMPOSSIBLE_ERROR) &&
!message.match?(/#{Regexp.quote(dependency.name)}/i)
# Sometimes pip-tools gets confused and can't work around
# sub-dependency incompatibilities. Ignore those cases.
return nil
end

if error.message.match?(GIT_REFERENCE_NOT_FOUND_REGEX)
tag = error.message.match(GIT_REFERENCE_NOT_FOUND_REGEX).named_captures.fetch("tag")
constraints_section = error.message.split("Finding the best candidates:").first
if message.match?(GIT_REFERENCE_NOT_FOUND_REGEX)
tag = message.match(GIT_REFERENCE_NOT_FOUND_REGEX).named_captures.fetch("tag")
constraints_section = message.split("Finding the best candidates:").first
egg_regex = /#{Regexp.escape(tag)}#egg=(#{PYTHON_PACKAGE_NAME_REGEX})/
name_match = constraints_section.scan(egg_regex)

Expand All @@ -166,15 +172,15 @@ def handle_pip_compile_errors(error)
raise GitDependencyReferenceNotFound, "(unknown package at #{tag})"
end

if error.message.match?(GIT_DEPENDENCY_UNREACHABLE_REGEX)
url = error.message.match(GIT_DEPENDENCY_UNREACHABLE_REGEX)
.named_captures.fetch("url")
if message.match?(GIT_DEPENDENCY_UNREACHABLE_REGEX)
url = message.match(GIT_DEPENDENCY_UNREACHABLE_REGEX)
.named_captures.fetch("url")
raise GitDependenciesNotReachable, url
end

raise Dependabot::OutOfDisk if error.message.end_with?("[Errno 28] No space left on device")
raise Dependabot::OutOfDisk if message.end_with?("[Errno 28] No space left on device")

raise Dependabot::OutOfMemory if error.message.end_with?("MemoryError")
raise Dependabot::OutOfMemory if message.end_with?("MemoryError")

raise
end
Expand Down
8 changes: 6 additions & 2 deletions python/spec/dependabot/python/update_checker/pip_compile_version_resolver_spec.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -393,7 +393,9 @@
context "because it ran out of disk space" do
before do
allow(Dependabot::SharedHelpers)
.to receive(:run_shell_command)
.to receive(:run_shell_command).and_call_original
allow(Dependabot::SharedHelpers)
.to receive(:run_shell_command).with(a_string_matching(/pyenv exec pip-compile/), *any_args)
.and_raise(
Dependabot::SharedHelpers::HelperSubprocessFailed.new(
message: "OSError: [Errno 28] No space left on device",
Expand All @@ -411,7 +413,9 @@
context "because it ran out of memory" do
before do
allow(Dependabot::SharedHelpers)
.to receive(:run_shell_command)
.to receive(:run_shell_command).and_call_original
allow(Dependabot::SharedHelpers)
.to receive(:run_shell_command).with(a_string_matching(/pyenv exec pip-compile/), *any_args)
.and_raise(
Dependabot::SharedHelpers::HelperSubprocessFailed.new(
message: "MemoryError",
Expand Down

0 comments on commit 2101a2a

Please sign in to comment.