Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix multiple pip compile errors #8189

Merged
merged 6 commits into from
Oct 16, 2023
Merged

Fix multiple pip compile errors #8189

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
c5af5f7
Relax detection of compilation errors
deivid-rodriguez Oct 10, 2023
e0a3829
Pass message directly to error handling method
deivid-rodriguez Oct 13, 2023
7adec31
Move memoization out of fetch_ method
deivid-rodriguez Oct 13, 2023
204ab8a
Refactor error handle in pip-compile version resolver
deivid-rodriguez Oct 13, 2023
fe361fb
Copy extension compilation retry logic from updater checker into file…
deivid-rodriguez Oct 13, 2023
450aa61
Workaround PyYAML with new cython
deivid-rodriguez Oct 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 19 additions & 5 deletions python/lib/dependabot/python/file_parser.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,28 +74,42 @@ def requirement_dependencies
# probably blocked. Ignore it.
next if blocking_marker?(dep)

name = dep["name"]
file = dep["file"]
version = dep["version"]

requirements =
if lockfile_for_pip_compile_file?(dep["file"]) then []
if lockfile_for_pip_compile_file?(file) then []
else
[{
requirement: dep["requirement"],
file: Pathname.new(dep["file"]).cleanpath.to_path,
file: Pathname.new(file).cleanpath.to_path,
source: nil,
groups: group_from_filename(dep["file"])
groups: group_from_filename(file)
}]
end

# PyYAML < 6.0 will cause `pip-compile` to fail due to incompatiblity with Cython 3. Workaround it.
SharedHelpers.run_shell_command("pyenv exec pip install cython<3.0") if old_pyyaml?(name, version)

dependencies <<
Dependency.new(
name: normalised_name(dep["name"], dep["extras"]),
version: dep["version"]&.include?("*") ? nil : dep["version"],
name: normalised_name(name, dep["extras"]),
version: version&.include?("*") ? nil : version,
requirements: requirements,
package_manager: "pip"
)
end
dependencies
end

def old_pyyaml?(name, version)
major_version = version&.split(".")&.first
return false unless major_version

name == "pyyaml" && major_version < "6"
end

def group_from_filename(filename)
if filename.include?("dev") then ["dev-dependencies"]
else
Expand Down
66 changes: 43 additions & 23 deletions python/lib/dependabot/python/file_updater/pip_compile_file_updater.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,16 @@ class PipCompileFileUpdater
WARNINGS = /\s*# WARNING:.*\Z/m
UNSAFE_NOTE = /\s*# The following packages are considered to be unsafe.*\Z/m
RESOLVER_REGEX = /(?<=--resolver=)(\w+)/
NATIVE_COMPILATION_ERROR =
"pip._internal.exceptions.InstallationSubprocessError: Getting requirements to build wheel exited with 1"

attr_reader :dependencies, :dependency_files, :credentials

def initialize(dependencies:, dependency_files:, credentials:)
@dependencies = dependencies
@dependency_files = dependency_files
@credentials = credentials
@build_isolation = true
end

def updated_dependency_files
Expand Down Expand Up @@ -67,28 +70,7 @@ def compile_new_requirement_files
language_version_manager.install_required_python

filenames_to_compile.each do |filename|
# Shell out to pip-compile, generate a new set of requirements.
# This is slow, as pip-compile needs to do installs.
options = pip_compile_options(filename)
options_fingerprint = pip_compile_options_fingerprint(options)

name_part = "pyenv exec pip-compile " \
"#{options} -P " \
"#{dependency.name}"
fingerprint_name_part = "pyenv exec pip-compile " \
"#{options_fingerprint} -P " \
"<dependency_name>"

version_part = "#{dependency.version} #{filename}"
fingerprint_version_part = "<dependency_version> <filename>"

# Don't escape pyenv `dep-name==version` syntax
run_pip_compile_command(
"#{SharedHelpers.escape_command(name_part)}==" \
"#{SharedHelpers.escape_command(version_part)}",
allow_unsafe_shell_command: true,
fingerprint: "#{fingerprint_name_part}==#{fingerprint_version_part}"
)
compile_file(filename)
end

# Remove any .python-version file before parsing the reqs
Expand All @@ -108,6 +90,44 @@ def compile_new_requirement_files
end
end

def compile_file(filename)
# Shell out to pip-compile, generate a new set of requirements.
# This is slow, as pip-compile needs to do installs.
options = pip_compile_options(filename)
options_fingerprint = pip_compile_options_fingerprint(options)

name_part = "pyenv exec pip-compile " \
"#{options} -P " \
"#{dependency.name}"
fingerprint_name_part = "pyenv exec pip-compile " \
"#{options_fingerprint} -P " \
"<dependency_name>"

version_part = "#{dependency.version} #{filename}"
fingerprint_version_part = "<dependency_version> <filename>"

# Don't escape pyenv `dep-name==version` syntax
run_pip_compile_command(
"#{SharedHelpers.escape_command(name_part)}==" \
"#{SharedHelpers.escape_command(version_part)}",
allow_unsafe_shell_command: true,
fingerprint: "#{fingerprint_name_part}==#{fingerprint_version_part}"
)
rescue SharedHelpers::HelperSubprocessFailed => e
retry_count ||= 0
retry_count += 1
if compilation_error?(e) && retry_count <= 1
@build_isolation = false
retry
end

raise
end

def compilation_error?(error)
error.message.include?(NATIVE_COMPILATION_ERROR)
end

def update_manifest_files
dependency_files.filter_map do |file|
next unless file.name.end_with?(".in")
Expand Down Expand Up @@ -403,7 +423,7 @@ def pip_compile_options_fingerprint(options)
end

def pip_compile_options(filename)
options = ["--build-isolation"]
options = @build_isolation ? ["--build-isolation"] : ["--no-build-isolation"]
options += pip_compile_index_options

if (requirements_file = compiled_file_for_filename(filename))
Expand Down
136 changes: 71 additions & 65 deletions python/lib/dependabot/python/update_checker/pip_compile_version_resolver.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ class PipCompileVersionResolver
GIT_DEPENDENCY_UNREACHABLE_REGEX = /git clone --filter=blob:none --quiet (?<url>[^\s]+).* /
GIT_REFERENCE_NOT_FOUND_REGEX = /Did not find branch or tag '(?<tag>[^\n"]+)'/m
NATIVE_COMPILATION_ERROR =
"pip._internal.exceptions.InstallationSubprocessError: Command errored out with exit status 1:"
"pip._internal.exceptions.InstallationSubprocessError: Getting requirements to build wheel exited with 1"
# See https://packaging.python.org/en/latest/tutorials/packaging-projects/#configuring-metadata
PYTHON_PACKAGE_NAME_REGEX = /[A-Za-z0-9_\-]+/
RESOLUTION_IMPOSSIBLE_ERROR = "ResolutionImpossible"
Expand All @@ -43,17 +43,21 @@ def initialize(dependency:, dependency_files:, credentials:)
end

def latest_resolvable_version(requirement: nil)
@latest_resolvable_version_string ||= {}
return @latest_resolvable_version_string[requirement] if @latest_resolvable_version_string.key?(requirement)

version_string =
fetch_latest_resolvable_version_string(requirement: requirement)

version_string.nil? ? nil : Python::Version.new(version_string)
@latest_resolvable_version_string[requirement] ||=
version_string.nil? ? nil : Python::Version.new(version_string)
end

def resolvable?(version:)
@resolvable ||= {}
return @resolvable[version] if @resolvable.key?(version)

@resolvable[version] = if fetch_latest_resolvable_version_string(requirement: "==#{version}")
@resolvable[version] = if latest_resolvable_version(requirement: "==#{version}")
true
else
false
Expand All @@ -63,57 +67,59 @@ def resolvable?(version:)
private

def fetch_latest_resolvable_version_string(requirement:)
@latest_resolvable_version_string ||= {}
return @latest_resolvable_version_string[requirement] if @latest_resolvable_version_string.key?(requirement)
SharedHelpers.in_a_temporary_directory do
SharedHelpers.with_git_configured(credentials: credentials) do
write_temporary_dependency_files(updated_req: requirement)
language_version_manager.install_required_python

@latest_resolvable_version_string[requirement] ||=
SharedHelpers.in_a_temporary_directory do
SharedHelpers.with_git_configured(credentials: credentials) do
write_temporary_dependency_files(updated_req: requirement)
language_version_manager.install_required_python

filenames_to_compile.each do |filename|
# Shell out to pip-compile.
# This is slow, as pip-compile needs to do installs.
options = pip_compile_options(filename)
options_fingerprint = pip_compile_options_fingerprint(options)

run_pip_compile_command(
"pyenv exec pip-compile -v #{options} -P #{dependency.name} #{filename}",
fingerprint: "pyenv exec pip-compile -v #{options_fingerprint} -P <dependency_name> <filename>"
)

next if dependency.top_level?

# Run pip-compile a second time for transient dependencies
# to make sure we do not update dependencies that are
# superfluous. pip-compile does not detect these when
# updating a specific dependency with the -P option.
# Running pip-compile a second time will automatically remove
# superfluous dependencies. Dependabot then marks those with
# update_not_possible.
write_original_manifest_files
run_pip_compile_command(
"pyenv exec pip-compile #{options} #{filename}",
fingerprint: "pyenv exec pip-compile #{options_fingerprint} <filename>"
)
end

# Remove any .python-version file before parsing the reqs
FileUtils.remove_entry(".python-version", true)

parse_updated_files
end
rescue SharedHelpers::HelperSubprocessFailed => e
retry_count ||= 0
retry_count += 1
if compilation_error?(e) && retry_count <= 1
@build_isolation = false
retry
filenames_to_compile.each do |filename|
return nil unless compile_file(filename)
end

handle_pip_compile_errors(e)
# Remove any .python-version file before parsing the reqs
FileUtils.remove_entry(".python-version", true)

parse_updated_files
end
end
end

def compile_file(filename)
# Shell out to pip-compile.
# This is slow, as pip-compile needs to do installs.
options = pip_compile_options(filename)
options_fingerprint = pip_compile_options_fingerprint(options)

run_pip_compile_command(
"pyenv exec pip-compile -v #{options} -P #{dependency.name} #{filename}",
fingerprint: "pyenv exec pip-compile -v #{options_fingerprint} -P <dependency_name> <filename>"
)

return true if dependency.top_level?

# Run pip-compile a second time for transient dependencies
# to make sure we do not update dependencies that are
# superfluous. pip-compile does not detect these when
# updating a specific dependency with the -P option.
# Running pip-compile a second time will automatically remove
# superfluous dependencies. Dependabot then marks those with
# update_not_possible.
write_original_manifest_files
run_pip_compile_command(
"pyenv exec pip-compile #{options} #{filename}",
fingerprint: "pyenv exec pip-compile #{options_fingerprint} <filename>"
)

true
rescue SharedHelpers::HelperSubprocessFailed => e
retry_count ||= 0
retry_count += 1
if compilation_error?(e) && retry_count <= 1
@build_isolation = false
retry
end

handle_pip_compile_errors(e.message)
end

def compilation_error?(error)
Expand All @@ -122,23 +128,23 @@ def compilation_error?(error)

# rubocop:disable Metrics/AbcSize
# rubocop:disable Metrics/PerceivedComplexity
def handle_pip_compile_errors(error)
if error.message.include?(RESOLUTION_IMPOSSIBLE_ERROR)
def handle_pip_compile_errors(message)
if message.include?(RESOLUTION_IMPOSSIBLE_ERROR)
check_original_requirements_resolvable
# If the original requirements are resolvable but we get an
# incompatibility error after unlocking then it's likely to be
# due to problems with pip-compile's cascading resolution
return nil
end

if error.message.include?("UnsupportedConstraint")
if message.include?("UnsupportedConstraint")
# If there's an unsupported constraint, check if it existed
# previously (and raise if it did)
check_original_requirements_resolvable
end

if (error.message.include?('Command "python setup.py egg_info') ||
error.message.include?(
if (message.include?('Command "python setup.py egg_info') ||
message.include?(
"exit status 1: python setup.py egg_info"
)) &&
check_original_requirements_resolvable
Expand All @@ -147,16 +153,16 @@ def handle_pip_compile_errors(error)
return
end

if error.message.include?(RESOLUTION_IMPOSSIBLE_ERROR) &&
!error.message.match?(/#{Regexp.quote(dependency.name)}/i)
if message.include?(RESOLUTION_IMPOSSIBLE_ERROR) &&
!message.match?(/#{Regexp.quote(dependency.name)}/i)
# Sometimes pip-tools gets confused and can't work around
# sub-dependency incompatibilities. Ignore those cases.
return nil
end

if error.message.match?(GIT_REFERENCE_NOT_FOUND_REGEX)
tag = error.message.match(GIT_REFERENCE_NOT_FOUND_REGEX).named_captures.fetch("tag")
constraints_section = error.message.split("Finding the best candidates:").first
if message.match?(GIT_REFERENCE_NOT_FOUND_REGEX)
tag = message.match(GIT_REFERENCE_NOT_FOUND_REGEX).named_captures.fetch("tag")
constraints_section = message.split("Finding the best candidates:").first
egg_regex = /#{Regexp.escape(tag)}#egg=(#{PYTHON_PACKAGE_NAME_REGEX})/
name_match = constraints_section.scan(egg_regex)

Expand All @@ -166,15 +172,15 @@ def handle_pip_compile_errors(error)
raise GitDependencyReferenceNotFound, "(unknown package at #{tag})"
end

if error.message.match?(GIT_DEPENDENCY_UNREACHABLE_REGEX)
url = error.message.match(GIT_DEPENDENCY_UNREACHABLE_REGEX)
.named_captures.fetch("url")
if message.match?(GIT_DEPENDENCY_UNREACHABLE_REGEX)
url = message.match(GIT_DEPENDENCY_UNREACHABLE_REGEX)
.named_captures.fetch("url")
raise GitDependenciesNotReachable, url
end

raise Dependabot::OutOfDisk if error.message.end_with?("[Errno 28] No space left on device")
raise Dependabot::OutOfDisk if message.end_with?("[Errno 28] No space left on device")

raise Dependabot::OutOfMemory if error.message.end_with?("MemoryError")
raise Dependabot::OutOfMemory if message.end_with?("MemoryError")

raise
end
Expand Down
8 changes: 6 additions & 2 deletions python/spec/dependabot/python/update_checker/pip_compile_version_resolver_spec.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -393,7 +393,9 @@
context "because it ran out of disk space" do
before do
allow(Dependabot::SharedHelpers)
.to receive(:run_shell_command)
.to receive(:run_shell_command).and_call_original
allow(Dependabot::SharedHelpers)
.to receive(:run_shell_command).with(a_string_matching(/pyenv exec pip-compile/), *any_args)
.and_raise(
Dependabot::SharedHelpers::HelperSubprocessFailed.new(
message: "OSError: [Errno 28] No space left on device",
Expand All @@ -411,7 +413,9 @@
context "because it ran out of memory" do
before do
allow(Dependabot::SharedHelpers)
.to receive(:run_shell_command)
.to receive(:run_shell_command).and_call_original
allow(Dependabot::SharedHelpers)
.to receive(:run_shell_command).with(a_string_matching(/pyenv exec pip-compile/), *any_args)
.and_raise(
Dependabot::SharedHelpers::HelperSubprocessFailed.new(
message: "MemoryError",
Expand Down