Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for centralized package manager abstraction for npm_and_yarn ecosystem #10862

Merged
Merged
81 changes: 53 additions & 28 deletions npm_and_yarn/lib/dependabot/npm_and_yarn/file_fetcher.rb
Original file line number Diff line number Diff line change
Expand Up @@ -182,71 +182,93 @@ def inferred_npmrc # rubocop:disable Metrics/PerceivedComplexity

sig { returns(T.nilable(T.any(Integer, String))) }
def npm_version
@npm_version ||= T.let(package_manager.setup("npm"), T.nilable(T.any(Integer, String)))
@npm_version ||= T.let(package_manager_helper.setup(NpmPackageManager::NAME), T.nilable(T.any(Integer, String)))
end

sig { returns(T.nilable(T.any(Integer, String))) }
def yarn_version
@yarn_version ||= T.let(package_manager.setup("yarn"), T.nilable(T.any(Integer, String)))
@yarn_version ||= T.let(
package_manager_helper.setup(YarnPackageManager::NAME),
T.nilable(T.any(Integer, String))
)
end

sig { returns(T.nilable(T.any(Integer, String))) }
def pnpm_version
@pnpm_version ||= T.let(package_manager.setup("pnpm"), T.nilable(T.any(Integer, String)))
@pnpm_version ||= T.let(
package_manager_helper.setup(PNPMPackageManager::NAME),
T.nilable(T.any(Integer, String))
)
end

sig { returns(PackageManager) }
def package_manager
@package_manager ||= T.let(PackageManager.new(
parsed_package_json,
lockfiles: { npm: package_lock || shrinkwrap, yarn: yarn_lock, pnpm: pnpm_lock }
), T.nilable(PackageManager))
sig { returns(PackageManagerHelper) }
def package_manager_helper
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We change the naming as package_manager_helper to reduce confusion.

  • This class was used as utility to install package manager for file_fetcher
  • We created Package Manager for each npm, yarn and pnpm and the helper is going to also help to create instance for package manager.

@package_manager_helper ||= T.let(
PackageManagerHelper.new(
parsed_package_json,
lockfiles: lockfiles
), T.nilable(PackageManagerHelper)
)
end

sig { returns(T::Hash[Symbol, T.nilable(Dependabot::DependencyFile)]) }
def lockfiles
{
npm: package_lock || shrinkwrap,
yarn: yarn_lock,
pnpm: pnpm_lock
}
end

sig { returns(DependencyFile) }
def package_json
@package_json ||= T.let(fetch_file_from_host("package.json"), T.nilable(DependencyFile))
@package_json ||= T.let(fetch_file_from_host(MANIFEST_FILENAME), T.nilable(DependencyFile))
end

sig { returns(T.nilable(DependencyFile)) }
def package_lock
return @package_lock if defined?(@package_lock)

@package_lock ||= T.let(fetch_file_if_present("package-lock.json"), T.nilable(DependencyFile))
@package_lock ||= T.let(fetch_file_if_present(NpmPackageManager::LOCKFILE_NAME), T.nilable(DependencyFile))
end

sig { returns(T.nilable(DependencyFile)) }
def yarn_lock
return @yarn_lock if defined?(@yarn_lock)

@yarn_lock ||= T.let(fetch_file_if_present("yarn.lock"), T.nilable(DependencyFile))
@yarn_lock ||= T.let(fetch_file_if_present(YarnPackageManager::LOCKFILE_NAME), T.nilable(DependencyFile))
end

sig { returns(T.nilable(DependencyFile)) }
def pnpm_lock
return @pnpm_lock if defined?(@pnpm_lock)

@pnpm_lock ||= T.let(fetch_file_if_present("pnpm-lock.yaml"), T.nilable(DependencyFile))
@pnpm_lock ||= T.let(fetch_file_if_present(PNPMPackageManager::LOCKFILE_NAME), T.nilable(DependencyFile))
end

sig { returns(T.nilable(DependencyFile)) }
def shrinkwrap
return @shrinkwrap if defined?(@shrinkwrap)

@shrinkwrap ||= T.let(fetch_file_if_present("npm-shrinkwrap.json"), T.nilable(DependencyFile))
@shrinkwrap ||= T.let(
fetch_file_if_present(
NpmPackageManager::SHRINKWRAP_LOCKFILE_NAME
),
T.nilable(DependencyFile)
)
end

sig { returns(T.nilable(DependencyFile)) }
def npmrc
return @npmrc if defined?(@npmrc)

@npmrc ||= T.let(fetch_support_file(".npmrc"), T.nilable(DependencyFile))
@npmrc ||= T.let(fetch_support_file(NpmPackageManager::RC_FILENAME), T.nilable(DependencyFile))

return @npmrc if @npmrc || directory == "/"

# Loop through parent directories looking for an npmrc
(1..directory.split("/").count).each do |i|
@npmrc = fetch_file_from_host(("../" * i) + ".npmrc")
@npmrc = fetch_file_from_host(("../" * i) + NpmPackageManager::RC_FILENAME)
.tap { |f| f.support_file = true }
break if @npmrc
rescue Dependabot::DependencyFileNotFound
Expand All @@ -261,13 +283,13 @@ def npmrc
def yarnrc
return @yarnrc if defined?(@yarnrc)

@yarnrc ||= T.let(fetch_support_file(".yarnrc"), T.nilable(DependencyFile))
@yarnrc ||= T.let(fetch_support_file(YarnPackageManager::RC_FILENAME), T.nilable(DependencyFile))

return @yarnrc if @yarnrc || directory == "/"

# Loop through parent directories looking for an yarnrc
(1..directory.split("/").count).each do |i|
@yarnrc = fetch_file_from_host(("../" * i) + ".yarnrc")
@yarnrc = fetch_file_from_host(("../" * i) + YarnPackageManager::RC_FILENAME)
.tap { |f| f.support_file = true }
break if @yarnrc
rescue Dependabot::DependencyFileNotFound
Expand All @@ -280,21 +302,24 @@ def yarnrc

sig { returns(T.nilable(DependencyFile)) }
def yarnrc_yml
@yarnrc_yml ||= T.let(fetch_support_file(".yarnrc.yml"), T.nilable(DependencyFile))
@yarnrc_yml ||= T.let(fetch_support_file(YarnPackageManager::RC_YML_FILENAME), T.nilable(DependencyFile))
end

sig { returns(T.nilable(DependencyFile)) }
def pnpm_workspace_yaml
return @pnpm_workspace_yaml if defined?(@pnpm_workspace_yaml)

@pnpm_workspace_yaml = T.let(fetch_support_file("pnpm-workspace.yaml"), T.nilable(DependencyFile))
@pnpm_workspace_yaml = T.let(
fetch_support_file(PNPMPackageManager::PNPM_WS_YML_FILENAME),
T.nilable(DependencyFile)
)
end

sig { returns(T.nilable(DependencyFile)) }
def lerna_json
return @lerna_json if defined?(@lerna_json)

@lerna_json = T.let(fetch_support_file("lerna.json"), T.nilable(DependencyFile))
@lerna_json = T.let(fetch_support_file(LERNA_JSON_FILENAME), T.nilable(DependencyFile))
end

sig { returns(T::Array[DependencyFile]) }
Expand Down Expand Up @@ -329,7 +354,7 @@ def path_dependencies(fetched_files)
filename = path
# NPM/Yarn support loading path dependencies from tarballs:
# https://docs.npmjs.com/cli/pack.html
filename = File.join(filename, "package.json") unless filename.end_with?(".tgz", ".tar", ".tar.gz")
filename = File.join(filename, MANIFEST_FILENAME) unless filename.end_with?(".tgz", ".tar", ".tar.gz")
cleaned_name = Pathname.new(filename).cleanpath.to_path
next if fetched_files.map(&:name).include?(cleaned_name)

Expand Down Expand Up @@ -380,7 +405,7 @@ def path_dependency_details(fetched_files)
# rubocop:disable Metrics/AbcSize
sig { params(file: DependencyFile).returns(T::Array[[String, String]]) }
def path_dependency_details_from_manifest(file)
return [] unless file.name.end_with?("package.json")
return [] unless file.name.end_with?(MANIFEST_FILENAME)

current_dir = file.name.rpartition("/").first
current_dir = nil if current_dir == ""
Expand Down Expand Up @@ -471,9 +496,9 @@ def fetch_lerna_packages_from_path(path)
return [] unless package_json

[package_json] + [
fetch_file_if_present(File.join(path, "package-lock.json")),
fetch_file_if_present(File.join(path, "yarn.lock")),
fetch_file_if_present(File.join(path, "npm-shrinkwrap.json"))
fetch_file_if_present(File.join(path, NpmPackageManager::LOCKFILE_NAME)),
fetch_file_if_present(File.join(path, YarnPackageManager::LOCKFILE_NAME)),
fetch_file_if_present(File.join(path, NpmPackageManager::SHRINKWRAP_LOCKFILE_NAME))
]
end

Expand Down Expand Up @@ -542,7 +567,7 @@ def recursive_find_directories(glob, prefix = "")

sig { params(workspace: String).returns(T.nilable(DependencyFile)) }
def fetch_package_json_if_present(workspace)
file = File.join(workspace, "package.json")
file = File.join(workspace, MANIFEST_FILENAME)

begin
fetch_file_from_host(file)
Expand Down Expand Up @@ -635,4 +660,4 @@ def parsed_lerna_json
end

Dependabot::FileFetchers
.register("npm_and_yarn", Dependabot::NpmAndYarn::FileFetcher)
.register(Dependabot::NpmAndYarn::ECOSYSTEM, Dependabot::NpmAndYarn::FileFetcher)
93 changes: 85 additions & 8 deletions npm_and_yarn/lib/dependabot/npm_and_yarn/file_parser.rb
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

module Dependabot
module NpmAndYarn
class FileParser < Dependabot::FileParsers::Base
class FileParser < Dependabot::FileParsers::Base # rubocop:disable Metrics/ClassLength
extend T::Sig

require "dependabot/file_parsers/base/dependency_set"
Expand Down Expand Up @@ -78,8 +78,82 @@ def parse
end
end

sig { returns(Ecosystem) }
def ecosystem
@ecosystem ||= T.let(
Ecosystem.new(
name: ECOSYSTEM,
package_manager: package_manager_helper.package_manager
),
T.nilable(Ecosystem)
)
end
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This method is used in dependency_snapshot to acquire information related to ecosystem, package manager and in upcoming PRs is going to be language information. That is also used for deprecation and unsupported checks however currently for npm_and_yarn they are disabled.


private

sig { returns(PackageManagerHelper) }
def package_manager_helper
@package_manager_helper ||= T.let(
PackageManagerHelper.new(
parsed_package_json,
lockfiles: lockfiles
), T.nilable(PackageManagerHelper)
)
end

sig { returns(T::Hash[Symbol, T.nilable(Dependabot::DependencyFile)]) }
def lockfiles
{
npm: package_lock || shrinkwrap,
yarn: yarn_lock,
pnpm: pnpm_lock
}
end

sig { returns(T.untyped) }
def parsed_package_json
JSON.parse(T.must(package_json.content))
rescue JSON::ParserError
raise Dependabot::DependencyFileNotParseable, package_json.path
end

sig { returns(Dependabot::DependencyFile) }
def package_json
# Declare the instance variable with T.let and the correct type
@package_json ||= T.let(
T.must(dependency_files.find { |f| f.name == MANIFEST_FILENAME }),
T.nilable(Dependabot::DependencyFile)
)
end

sig { returns(T.nilable(Dependabot::DependencyFile)) }
def shrinkwrap
@shrinkwrap ||= T.let(dependency_files.find do |f|
f.name == NpmPackageManager::SHRINKWRAP_LOCKFILE_NAME
end, T.nilable(Dependabot::DependencyFile))
end

sig { returns(T.nilable(Dependabot::DependencyFile)) }
def package_lock
@package_lock ||= T.let(dependency_files.find do |f|
f.name == NpmPackageManager::LOCKFILE_NAME
end, T.nilable(Dependabot::DependencyFile))
end

sig { returns(T.nilable(Dependabot::DependencyFile)) }
def yarn_lock
@yarn_lock ||= T.let(dependency_files.find do |f|
f.name == YarnPackageManager::LOCKFILE_NAME
end, T.nilable(Dependabot::DependencyFile))
end

sig { returns(T.nilable(Dependabot::DependencyFile)) }
def pnpm_lock
@pnpm_lock ||= T.let(dependency_files.find do |f|
f.name == PNPMPackageManager::LOCKFILE_NAME
end, T.nilable(Dependabot::DependencyFile))
end

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

To create an instance from the package manager helper, we require dependency_files. This helper is already being used in the file_fetcher, and to avoid duplication and maintain consistency, we're using the same helper in the file_parser.

sig { returns(Dependabot::FileParsers::Base::DependencySet) }
def manifest_dependencies
dependency_set = DependencySet.new
Expand Down Expand Up @@ -154,7 +228,7 @@ def build_dependency(file:, type:, name:, requirement:)
Dependency.new(
name: name,
version: converted_version,
package_manager: "npm_and_yarn",
package_manager: ECOSYSTEM,
requirements: [{
requirement: requirement_for(requirement),
file: file.name,
Expand All @@ -166,7 +240,10 @@ def build_dependency(file:, type:, name:, requirement:)

sig { override.void }
def check_required_files
raise DependencyFileNotFound.new(nil, "package.json not found.") unless get_original_file("package.json")
return if get_original_file(MANIFEST_FILENAME)

raise DependencyFileNotFound.new(nil,
"#{MANIFEST_FILENAME} not found.")
end

sig { params(requirement: String).returns(T::Boolean) }
Expand All @@ -186,7 +263,7 @@ def local_path?(requirement)

sig { params(requirement: String).returns(T::Boolean) }
def alias_package?(requirement)
requirement.start_with?("npm:")
requirement.start_with?("#{NpmPackageManager::NAME}:")
end

sig { params(requirement: String).returns(T::Boolean) }
Expand All @@ -208,7 +285,7 @@ def git_url_with_semver?(requirement)

sig { params(name: String).returns(T::Boolean) }
def aliased_package_name?(name)
name.include?("@npm:")
name.include?("@#{NpmPackageManager::NAME}:")
end

sig { returns(T::Array[String]) }
Expand Down Expand Up @@ -370,8 +447,8 @@ def support_package_files
def sub_package_files
return T.must(@sub_package_files) if defined?(@sub_package_files)

files = dependency_files.select { |f| f.name.end_with?("package.json") }
.reject { |f| f.name == "package.json" }
files = dependency_files.select { |f| f.name.end_with?(MANIFEST_FILENAME) }
.reject { |f| f.name == MANIFEST_FILENAME }
.reject { |f| f.name.include?("node_modules/") }
@sub_package_files ||= T.let(files, T.nilable(T::Array[Dependabot::DependencyFile]))
end
Expand All @@ -380,7 +457,7 @@ def sub_package_files
def package_files
@package_files ||= T.let(
[
dependency_files.find { |f| f.name == "package.json" },
dependency_files.find { |f| f.name == MANIFEST_FILENAME },
*sub_package_files
].compact, T.nilable(T::Array[DependencyFile])
)
Expand Down
Loading
Loading