Merge pull request dependabot#2596 from dependabot/jurre/go-vendor

Add vendoring support for go_modules

Author: Pete Wagner

.gitignore

@@ -9,6 +9,7 @@ Gemfile.lock
 vendor
 !bundler/spec/fixtures/vendored_gems/vendor
 !common/spec/fixtures/projects/**/*/vendor
+!go_modules/spec/fixtures/projects/**/*
 .DS_Store
 *.pyc
 *git.store

go_modules/lib/dependabot/go_modules/file_updater.rb

@@ -3,6 +3,7 @@
 require "dependabot/shared_helpers"
 require "dependabot/file_updaters"
 require "dependabot/file_updaters/base"
+require "dependabot/file_updaters/vendor_updater"
 
 module Dependabot
   module GoModules
@@ -54,6 +55,12 @@ def updated_dependency_files
                 content: file_updater.updated_go_sum_content
               )
           end
+
+          vendor_updater.
+            updated_vendor_cache_files(base_directory: directory).
+            each do |file|
+            updated_files << file
+          end
         end
 
         raise "No files changed!" if updated_files.none?
@@ -81,16 +88,36 @@ def directory
         dependency_files.first.directory
       end
 
+      def vendor_dir
+        File.join(repo_contents_path, directory, "vendor")
+      end
+
+      def vendor_updater
+        Dependabot::FileUpdaters::VendorUpdater.new(
+          repo_contents_path: repo_contents_path,
+          vendor_dir: vendor_dir
+        )
+      end
+
       def file_updater
         @file_updater ||=
           GoModUpdater.new(
             dependencies: dependencies,
             credentials: credentials,
             repo_contents_path: repo_contents_path,
             directory: directory,
-            tidy: !@repo_contents_stub && options.fetch(:go_mod_tidy, false)
+            options: { tidy: tidy?, vendor: vendor? }
           )
       end
+
+      def tidy?
+        !@repo_contents_stub && options.fetch(:go_mod_tidy, false)
+      end
+
+      def vendor?
+        File.exist?(File.join(vendor_dir, "modules.txt")) &&
+          options.fetch(:go_mod_vendor, false)
+      end
     end
   end
 end

go_modules/lib/dependabot/go_modules/file_updater/go_mod_updater.rb

@@ -26,12 +26,13 @@ class GoModUpdater
         ].freeze
 
         def initialize(dependencies:, credentials:, repo_contents_path:,
-                       directory:, tidy:)
+                       directory:, options:)
           @dependencies = dependencies
           @credentials = credentials
           @repo_contents_path = repo_contents_path
           @directory = directory
-          @tidy = tidy
+          @tidy = options.fetch(:tidy, false)
+          @vendor = options.fetch(:vendor, false)
         end
 
         def updated_go_mod_content
@@ -51,7 +52,7 @@ def updated_files
           @updated_files ||= update_files
         end
 
-        def update_files
+        def update_files # rubocop:disable Metrics/AbcSize
           in_repo_path do
             # Map paths in local replace directives to path hashes
 
@@ -71,6 +72,7 @@ def update_files
             # Then run `go get` to pick up other changes to the file caused by
             # the upgrade
             run_go_get
+            run_go_vendor
             run_go_mod_tidy
 
             # At this point, the go.mod returned from run_go_get contains the
@@ -111,6 +113,14 @@ def run_go_mod_tidy
           handle_subprocess_error(stderr) unless status.success?
         end
 
+        def run_go_vendor
+          return unless vendor?
+
+          command = "go mod vendor"
+          _, stderr, status = Open3.capture3(ENVIRONMENT, command)
+          handle_subprocess_error(stderr) unless status.success?
+        end
+
         def update_go_mod(dependencies)
           deps = dependencies.map do |dep|
             {
@@ -273,6 +283,10 @@ def write_go_mod(body)
         def tidy?
           !!@tidy
         end
+
+        def vendor?
+          !!@vendor
+        end
       end
     end
   end

go_modules/spec/dependabot/go_modules/file_updater/go_mod_updater_spec.rb

@@ -17,7 +17,7 @@
       }],
       repo_contents_path: repo_contents_path,
       directory: "/",
-      tidy: tidy
+      options: { tidy: tidy, vendor: false }
     )
   end
 

go_modules/spec/dependabot/go_modules/file_updater_spec.rb

@@ -22,6 +22,7 @@
   let(:files) { [go_mod, go_sum] }
   let(:project_name) { "go_sum" }
   let(:repo_contents_path) { build_tmp_repo(project_name) }
+  let(:vendor) { false }
 
   let(:credentials) do
     [{
@@ -31,7 +32,7 @@
       "password" => "token"
     }]
   end
-  let(:options) { {} }
+  let(:options) { { go_mod_vendor: vendor } }
 
   let(:go_mod) do
     Dependabot::DependencyFile.new(name: "go.mod", content: go_mod_body)
@@ -93,7 +94,7 @@
     end
 
     context "options" do
-      let(:options) { { go_mod_tidy: true } }
+      let(:options) { { go_mod_tidy: true, go_mod_vendor: vendor } }
       let(:dummy_updater) do
         instance_double(
           Dependabot::GoModules::FileUpdater::GoModUpdater,
@@ -110,11 +111,24 @@
             credentials: credentials,
             repo_contents_path: repo_contents_path,
             directory: "/",
-            tidy: true
+            options: { tidy: true, vendor: false }
           ).and_return(dummy_updater)
 
         updater.updated_dependency_files
       end
+
+      context "vendor option is passed but vendor directory not checked in" do
+        let(:vendor) { true }
+
+        it "does not includes the vendored files" do
+          expect(updater.updated_dependency_files.map(&:name)).to match_array(
+            %w(
+              go.mod
+              go.sum
+            )
+          )
+        end
+      end
     end
 
     context "without a go.sum" do
@@ -163,5 +177,93 @@
         expect(updated_files.find { |f| f.name == "go.sum" }).to_not be_nil
       end
     end
+
+    context "vendoring" do
+      let(:project_name) { "vendor" }
+      let(:vendor) { true }
+
+      let(:dependency_name) { "github.com/pkg/errors" }
+      let(:dependency_version) { "v0.9.1" }
+      let(:dependency_previous_version) { "v0.8.0" }
+      let(:requirements) do
+        [{
+          file: "go.mod",
+          requirement: dependency_version,
+          groups: [],
+          source: {
+            type: "default",
+            source: "github.com/pkg"
+          }
+        }]
+      end
+      let(:previous_requirements) do
+        [{
+          file: "go.mod",
+          requirement: dependency_previous_version,
+          groups: [],
+          source: {
+            type: "default",
+            source: "github.com/pkg"
+          }
+        }]
+      end
+
+      it "updates the go.mod" do
+        expect(go_mod_body).to include("github.com/pkg/errors v0.8.0")
+
+        updater.updated_dependency_files
+
+        go_mod_file = updater.updated_dependency_files.find do |file|
+          file.name == "go.mod"
+        end
+
+        expect(go_mod_file.content).to include "github.com/pkg/errors v0.9.1"
+      end
+
+      it "includes the vendored files" do
+        expect(updater.updated_dependency_files.map(&:name)).to match_array(
+          %w(
+            go.mod
+            go.sum
+            vendor/github.com/pkg/errors/.travis.yml
+            vendor/github.com/pkg/errors/Makefile
+            vendor/github.com/pkg/errors/README.md
+            vendor/github.com/pkg/errors/errors.go
+            vendor/github.com/pkg/errors/go113.go
+            vendor/github.com/pkg/errors/stack.go
+            vendor/modules.txt
+          )
+        )
+      end
+
+      it "updates the vendor/modules.txt file to the right version" do
+        modules_file = updater.updated_dependency_files.find do |file|
+          file.name == "vendor/modules.txt"
+        end
+
+        expect(modules_file.content).
+          to_not include "github.com/pkg/errors v0.8.0"
+        expect(modules_file.content).to include "github.com/pkg/errors v0.9.1"
+      end
+
+      it "includes the new source code" do
+        # Sample to verify the source code matches:
+        # https://github.com/pkg/errors/compare/v0.8.0...v0.9.1
+        stack_file = updater.updated_dependency_files.find do |file|
+          file.name == "vendor/github.com/pkg/errors/stack.go"
+        end
+
+        expect(stack_file.content).to include(
+          <<~LINE
+            Format formats the stack of Frames according to the fmt.Formatter interface.
+          LINE
+        )
+        expect(stack_file.content).to_not include(
+          <<~LINE
+            segments from the beginning of the file path until the number of path
+          LINE
+        )
+      end
+    end
   end
 end

go_modules/spec/fixtures/projects/vendor/go.mod

@@ -0,0 +1,5 @@
+module github.com/dependabot/vgotest
+
+go 1.12
+
+require github.com/pkg/errors v0.8.0

go_modules/spec/fixtures/projects/vendor/go.sum

@@ -0,0 +1,2 @@
+github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

go_modules/spec/fixtures/projects/vendor/main.go

@@ -0,0 +1,8 @@
+package main
+
+import (
+	_ "github.com/pkg/errors"
+)
+
+func main() {
+}