Merge remote-tracking branch 'origin/main' into bundler-2-spike

.github/workflows/ci.yml

@@ -101,4 +101,4 @@ jobs:
           docker run --rm "$CORE_CI_IMAGE" bash -c "cd /opt/npm_and_yarn && yarn test"
       - name: Run ${{ matrix.suite }} tests with rspec
         run: |
-          docker run --rm "$CORE_CI_IMAGE" bash -c "cd /home/dependabot/dependabot-core/${{ matrix.suite }} && bundle exec rspec spec"
+          docker run --env "CI=true" --rm "$CORE_CI_IMAGE" bash -c "cd /home/dependabot/dependabot-core/${{ matrix.suite }} && bundle exec rspec spec"

.gitignore

@@ -9,6 +9,7 @@ Gemfile.lock
 vendor
 !bundler/spec/fixtures/vendored_gems/vendor
 !common/spec/fixtures/projects/**/*/vendor
+!go_modules/spec/fixtures/projects/**/*
 .DS_Store
 *.pyc
 *git.store
@@ -24,3 +25,4 @@ vendor
 /dry-run
 **/bin/helper
 /.core-bash_history
+coverage/

CHANGELOG.md

@@ -1,3 +1,15 @@
+## v0.122.0, 7 October 2020
+
+- Add experimental support for `go mod vendor`
+- Enable code coverage reporting of dependabot-core
+
+## v0.121.1, 7 October 2020
+
+- Configure git when creating a temp repo for gomod updates
+- Bump jest from 26.5.0 to 26.5.2 in /npm_and_yarn/helpers
+- Bump poetry from 1.1.1 to 1.1.2 in /python/helpers
+- Refactor: reusable VendorDependencies object
+
 ## v0.121.0, 6 October 2020
 
 - Add experimental support for `go mod tidy`

bin/dry-run.rb

@@ -99,7 +99,8 @@
   clone: false,
   lockfile_only: false,
   requirements_update_strategy: nil,
-  commit: nil
+  commit: nil,
+  updater_options: {},
 }
 
 unless ENV["LOCAL_GITHUB_ACCESS_TOKEN"].to_s.strip.empty?
@@ -159,6 +160,16 @@
   opts.on("--clone", "clone the repo") do |_value|
     $options[:clone] = true
   end
+
+  opts_opt_desc = "Comma separated list of updater options, "\
+                  "available options depend on PACKAGE_MANAGER"
+  opts.on("--updater-options OPTIONS", opts_opt_desc) do |value|
+    $options[:updater_options] = Hash[
+                                   value.split(",").map do |o|
+                                     [o.strip.downcase.to_sym, true]
+                                   end
+                                 ]
+  end
 end
 
 option_parse.parse!
@@ -393,7 +404,8 @@ def file_updater_for(dependencies)
     dependencies: dependencies,
     dependency_files: $files,
     repo_contents_path: $repo_contents_path,
-    credentials: $options[:credentials]
+    credentials: $options[:credentials],
+    options: $options[:updater_options],
   )
 end
 

common/dependabot-common.gemspec

@@ -40,7 +40,9 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 13"
   spec.add_development_dependency "rspec", "~> 3.8"
   spec.add_development_dependency "rspec-its", "~> 1.2"
-  spec.add_development_dependency "rubocop", "~> 0.92.0"
+  spec.add_development_dependency "rubocop", "~> 0.93.0"
+  spec.add_development_dependency "simplecov", "~> 0.19.0"
+  spec.add_development_dependency "simplecov-console", "~> 0.7.2"
   spec.add_development_dependency "vcr", "6.0.0"
   spec.add_development_dependency "webmock", "~> 3.4"
 

common/lib/dependabot/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Dependabot
-  VERSION = "0.121.0"
+  VERSION = "0.122.0"
 end

common/spec/spec_helper.rb

@@ -4,9 +4,28 @@
 require "webmock/rspec"
 require "vcr"
 require "byebug"
+require "simplecov"
+require "simplecov-console"
 
 require_relative "dummy_package_manager/dummy"
 
+SimpleCov::Formatter::Console.output_style = "block"
+SimpleCov.formatter = if ENV["CI"]
+                        SimpleCov::Formatter::Console
+                      else
+                        SimpleCov::Formatter::HTMLFormatter
+                      end
+
+SimpleCov.start do
+  add_filter "/spec/"
+
+  enable_coverage :branch
+  minimum_coverage line: 80, branch: 70
+  # TODO: Enable minimum coverage per file once outliers have been increased
+  # minimum_coverage_by_file 80
+  refuse_coverage_drop
+end
+
 RSpec.configure do |config|
   config.color = true
   config.order = :rand

go_modules/lib/dependabot/go_modules/file_updater.rb

@@ -3,6 +3,7 @@
 require "dependabot/shared_helpers"
 require "dependabot/file_updaters"
 require "dependabot/file_updaters/base"
+require "dependabot/file_updaters/vendor_updater"
 
 module Dependabot
   module GoModules
@@ -54,6 +55,12 @@ def updated_dependency_files
                 content: file_updater.updated_go_sum_content
               )
           end
+
+          vendor_updater.
+            updated_vendor_cache_files(base_directory: directory).
+            each do |file|
+            updated_files << file
+          end
         end
 
         raise "No files changed!" if updated_files.none?
@@ -81,16 +88,36 @@ def directory
         dependency_files.first.directory
       end
 
+      def vendor_dir
+        File.join(repo_contents_path, directory, "vendor")
+      end
+
+      def vendor_updater
+        Dependabot::FileUpdaters::VendorUpdater.new(
+          repo_contents_path: repo_contents_path,
+          vendor_dir: vendor_dir
+        )
+      end
+
       def file_updater
         @file_updater ||=
           GoModUpdater.new(
             dependencies: dependencies,
             credentials: credentials,
             repo_contents_path: repo_contents_path,
             directory: directory,
-            tidy: !@repo_contents_stub && options.fetch(:go_mod_tidy, false)
+            options: { tidy: tidy?, vendor: vendor? }
           )
       end
+
+      def tidy?
+        !@repo_contents_stub && options.fetch(:go_mod_tidy, false)
+      end
+
+      def vendor?
+        File.exist?(File.join(vendor_dir, "modules.txt")) &&
+          options.fetch(:go_mod_vendor, false)
+      end
     end
   end
 end

go_modules/lib/dependabot/go_modules/file_updater/go_mod_updater.rb

@@ -26,12 +26,13 @@ class GoModUpdater
         ].freeze
 
         def initialize(dependencies:, credentials:, repo_contents_path:,
-                       directory:, tidy:)
+                       directory:, options:)
           @dependencies = dependencies
           @credentials = credentials
           @repo_contents_path = repo_contents_path
           @directory = directory
-          @tidy = tidy
+          @tidy = options.fetch(:tidy, false)
+          @vendor = options.fetch(:vendor, false)
         end
 
         def updated_go_mod_content
@@ -51,7 +52,7 @@ def updated_files
           @updated_files ||= update_files
         end
 
-        def update_files
+        def update_files # rubocop:disable Metrics/AbcSize
           in_repo_path do
             # Map paths in local replace directives to path hashes
 
@@ -71,6 +72,7 @@ def update_files
             # Then run `go get` to pick up other changes to the file caused by
             # the upgrade
             run_go_get
+            run_go_vendor
             run_go_mod_tidy
 
             # At this point, the go.mod returned from run_go_get contains the
@@ -111,6 +113,14 @@ def run_go_mod_tidy
           handle_subprocess_error(stderr) unless status.success?
         end
 
+        def run_go_vendor
+          return unless vendor?
+
+          command = "go mod vendor"
+          _, stderr, status = Open3.capture3(ENVIRONMENT, command)
+          handle_subprocess_error(stderr) unless status.success?
+        end
+
         def update_go_mod(dependencies)
           deps = dependencies.map do |dep|
             {
@@ -273,6 +283,10 @@ def write_go_mod(body)
         def tidy?
           !!@tidy
         end
+
+        def vendor?
+          !!@vendor
+        end
       end
     end
   end

go_modules/spec/dependabot/go_modules/file_updater/go_mod_updater_spec.rb

@@ -17,7 +17,7 @@
       }],
       repo_contents_path: repo_contents_path,
       directory: "/",
-      tidy: tidy
+      options: { tidy: tidy, vendor: false }
     )
   end
 

go_modules/spec/dependabot/go_modules/file_updater_spec.rb

@@ -22,6 +22,7 @@
   let(:files) { [go_mod, go_sum] }
   let(:project_name) { "go_sum" }
   let(:repo_contents_path) { build_tmp_repo(project_name) }
+  let(:vendor) { false }
 
   let(:credentials) do
     [{
@@ -31,7 +32,7 @@
       "password" => "token"
     }]
   end
-  let(:options) { {} }
+  let(:options) { { go_mod_vendor: vendor } }
 
   let(:go_mod) do
     Dependabot::DependencyFile.new(name: "go.mod", content: go_mod_body)
@@ -93,7 +94,7 @@
     end
 
     context "options" do
-      let(:options) { { go_mod_tidy: true } }
+      let(:options) { { go_mod_tidy: true, go_mod_vendor: vendor } }
       let(:dummy_updater) do
         instance_double(
           Dependabot::GoModules::FileUpdater::GoModUpdater,
@@ -110,11 +111,24 @@
             credentials: credentials,
             repo_contents_path: repo_contents_path,
             directory: "/",
-            tidy: true
+            options: { tidy: true, vendor: false }
           ).and_return(dummy_updater)
 
         updater.updated_dependency_files
       end
+
+      context "vendor option is passed but vendor directory not checked in" do
+        let(:vendor) { true }
+
+        it "does not includes the vendored files" do
+          expect(updater.updated_dependency_files.map(&:name)).to match_array(
+            %w(
+              go.mod
+              go.sum
+            )
+          )
+        end
+      end
     end
 
     context "without a go.sum" do
@@ -163,5 +177,93 @@
         expect(updated_files.find { |f| f.name == "go.sum" }).to_not be_nil
       end
     end
+
+    context "vendoring" do
+      let(:project_name) { "vendor" }
+      let(:vendor) { true }
+
+      let(:dependency_name) { "github.com/pkg/errors" }
+      let(:dependency_version) { "v0.9.1" }
+      let(:dependency_previous_version) { "v0.8.0" }
+      let(:requirements) do
+        [{
+          file: "go.mod",
+          requirement: dependency_version,
+          groups: [],
+          source: {
+            type: "default",
+            source: "github.com/pkg"
+          }
+        }]
+      end
+      let(:previous_requirements) do
+        [{
+          file: "go.mod",
+          requirement: dependency_previous_version,
+          groups: [],
+          source: {
+            type: "default",
+            source: "github.com/pkg"
+          }
+        }]
+      end
+
+      it "updates the go.mod" do
+        expect(go_mod_body).to include("github.com/pkg/errors v0.8.0")
+
+        updater.updated_dependency_files
+
+        go_mod_file = updater.updated_dependency_files.find do |file|
+          file.name == "go.mod"
+        end
+
+        expect(go_mod_file.content).to include "github.com/pkg/errors v0.9.1"
+      end
+
+      it "includes the vendored files" do
+        expect(updater.updated_dependency_files.map(&:name)).to match_array(
+          %w(
+            go.mod
+            go.sum
+            vendor/github.com/pkg/errors/.travis.yml
+            vendor/github.com/pkg/errors/Makefile
+            vendor/github.com/pkg/errors/README.md
+            vendor/github.com/pkg/errors/errors.go
+            vendor/github.com/pkg/errors/go113.go
+            vendor/github.com/pkg/errors/stack.go
+            vendor/modules.txt
+          )
+        )
+      end
+
+      it "updates the vendor/modules.txt file to the right version" do
+        modules_file = updater.updated_dependency_files.find do |file|
+          file.name == "vendor/modules.txt"
+        end
+
+        expect(modules_file.content).
+          to_not include "github.com/pkg/errors v0.8.0"
+        expect(modules_file.content).to include "github.com/pkg/errors v0.9.1"
+      end
+
+      it "includes the new source code" do
+        # Sample to verify the source code matches:
+        # https://github.com/pkg/errors/compare/v0.8.0...v0.9.1
+        stack_file = updater.updated_dependency_files.find do |file|
+          file.name == "vendor/github.com/pkg/errors/stack.go"
+        end
+
+        expect(stack_file.content).to include(
+          <<~LINE
+            Format formats the stack of Frames according to the fmt.Formatter interface.
+          LINE
+        )
+        expect(stack_file.content).to_not include(
+          <<~LINE
+            segments from the beginning of the file path until the number of path
+          LINE
+        )
+      end
+    end
   end
 end

go_modules/spec/fixtures/projects/vendor/go.mod

@@ -0,0 +1,5 @@
+module github.com/dependabot/vgotest
+
+go 1.12
+
+require github.com/pkg/errors v0.8.0