diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 11011ddb..5c1eabe1 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -1,12 +1,20 @@ # This configuration was generated by # `rubocop --auto-gen-config` -# on 2021-10-18 14:18:47 UTC using RuboCop version 1.22.1. +# on 2022-02-14 21:55:21 UTC using RuboCop version 1.24.1. # The point is for the user to remove these configuration records # one by one as the offenses are removed from the code base. # Note that changes in the inspected code, or installation of new # versions of RuboCop, may require this file to be generated again. -# Offense count: 8 +# Offense count: 1 +# Cop supports --auto-correct. +# Configuration parameters: Include. +# Include: **/*.gemspec +Gemspec/RequireMFA: + Exclude: + - 'modulesync.gemspec' + +# Offense count: 9 # Configuration parameters: IgnoredMethods, CountRepeatedAttributes. Metrics/AbcSize: Max: 60 @@ -14,19 +22,19 @@ Metrics/AbcSize: # Offense count: 2 # Configuration parameters: CountComments, CountAsOne. Metrics/ClassLength: - Max: 134 + Max: 186 -# Offense count: 4 +# Offense count: 5 # Configuration parameters: IgnoredMethods. Metrics/CyclomaticComplexity: Max: 15 -# Offense count: 14 +# Offense count: 17 # Configuration parameters: CountComments, CountAsOne, ExcludedMethods, IgnoredMethods. Metrics/MethodLength: Max: 34 -# Offense count: 3 +# Offense count: 4 # Configuration parameters: IgnoredMethods. Metrics/PerceivedComplexity: Max: 16 diff --git a/.simplecov b/.simplecov index 88620026..30cde509 100644 --- a/.simplecov +++ b/.simplecov @@ -33,7 +33,7 @@ SimpleCov.start do track_files '**/*.rb' end -if ENV['CODECOV'] +if ENV['CODECOV'] == 'yes' require 'simplecov-console' require 'codecov' diff --git a/features/execute.feature b/features/execute.feature new file mode 100644 index 00000000..cee91ca3 --- /dev/null +++ b/features/execute.feature @@ -0,0 +1,51 @@ +Feature: execute + Use ModuleSync to execute a custom script on each repositories + + Scenario: Cloning sourcecodes before running command when modules/ dir is empty + Given a basic setup with a puppet module "puppet-test" from "awesome" + Then the file "modules/awesome/puppet-test/metadata.json" should not exist + When I successfully run `msync exec --verbose -- /bin/true` + Then the stdout should contain "Cloning from 'file://" + And the file "modules/awesome/puppet-test/metadata.json" should exist + + @no-clobber + Scenario: No clones before running command when sourcecode have already been cloned + Then the file "modules/awesome/puppet-test/metadata.json" should exist + When I successfully run `msync exec --verbose /bin/true` + Then the stdout should not contain "Cloning from 'file://" + + @no-clobber + Scenario: When command run fails, fail fast if option defined + When I run `msync exec --verbose --fail-fast -- /bin/false` + Then the exit status should be 1 + And the stderr should contain: + """ + Command execution failed + """ + + @no-clobber + Scenario: When command run fails, run all and summarize errors if option fail-fast is not set + When I run `msync exec --verbose --no-fail-fast -- /bin/false` + Then the exit status should be 1 + And the stderr should contain: + """ + Error(s) during `execute` command: + * + """ + + Scenario: Show fail-fast default value in help + When I successfully run `msync help exec` + Then the stdout should contain: + """ + [--fail-fast], [--no-fail-fast] # Abort the run after a command execution failure + # Default: true + """ + + Scenario: Override fail-fast default value using config file + Given the global option "fail_fast" sets to "false" + When I successfully run `msync help exec` + Then the stdout should contain: + """ + [--fail-fast], [--no-fail-fast] # Abort the run after a command execution failure + """ + # NOTE: It seems there is a Thor bug here: default value is missing in help when sets to 'false' diff --git a/features/push.feature b/features/push.feature new file mode 100644 index 00000000..4e6dda0c --- /dev/null +++ b/features/push.feature @@ -0,0 +1,46 @@ +Feature: push + Push commits to remote + + Scenario: Push available commits to remote + Given a mocked git configuration + And a puppet module "puppet-test" from "awesome" + And a file named "managed_modules.yml" with: + """ + --- + puppet-test: + namespace: awesome + """ + And a file named "modulesync.yml" with: + """ + --- + branch: modulesync + """ + And a git_base option appended to "modulesync.yml" for local tests + And I successfully run `msync reset` + And I cd to "modules/awesome/puppet-test" + And I run `touch hello` + And I run `git add hello` + And I run `git commit -m'Hello!'` + And I cd to "~" + Then the puppet module "puppet-test" from "awesome" should have no commits made by "Aruba" + When I successfully run `msync push --verbose` + Then the puppet module "puppet-test" from "awesome" should have 1 commit made by "Aruba" in branch "modulesync" + + Scenario: Push command without a branch sets + Given a basic setup with a puppet module "puppet-test" from "awesome" + When I run `msync push --verbose` + Then the exit status should be 1 + And the stderr should contain: + """ + Error: 'branch' option is missing, please set it in configuration or in command line. + """ + + Scenario: Report the need to clone repositories if sourcecode was not cloned before + Given a basic setup with a puppet module "puppet-test" from "awesome" + And the global option "branch" sets to "modulesync" + When I run `msync push --verbose` + Then the exit status should be 1 + And the stderr should contain: + """ + puppet-test: Repository must be locally available before trying to push + """ diff --git a/features/reset.feature b/features/reset.feature new file mode 100644 index 00000000..df41da39 --- /dev/null +++ b/features/reset.feature @@ -0,0 +1,57 @@ +Feature: reset + Reset all repositories + + Scenario: Running first reset to clone repositories + Given a basic setup with a puppet module "puppet-test" from "awesome" + And the global option "branch" sets to "modulesync" + When I successfully run `msync reset --verbose` + Then the output should contain "Cloning from 'file://" + And the output should not contain "Hard-resetting any local changes to repository in" + + @no-clobber + Scenario: Reset when sourcecodes have already been cloned + Given the file "modules/awesome/puppet-test/metadata.json" should exist + And the global option "branch" sets to "modulesync" + When I successfully run `msync reset --verbose` + Then the output should not contain "Cloning from 'file://" + And the output should contain "Hard-resetting any local changes to repository in 'modules/awesome/puppet-test' from branch 'origin/master'" + + Scenario: Reset after an upstream file addition + Given a basic setup with a puppet module "puppet-test" from "awesome" + And the global option "branch" sets to "modulesync" + And I successfully run `msync reset` + Then the file "modules/awesome/puppet-test/hello" should not exist + When the puppet module "puppet-test" from "awesome" has a file named "hello" with: + """ + Hello + """ + When I successfully run `msync reset --verbose` + Then the output should contain "Hard-resetting any local changes to repository in 'modules/awesome/puppet-test' from branch 'origin/master'" + And the file "modules/awesome/puppet-test/hello" should exist + + Scenario: Reset after an upstream file addition in offline mode + Given a basic setup with a puppet module "puppet-test" from "awesome" + And the global option "branch" sets to "modulesync" + And I successfully run `msync reset` + Then the file "modules/awesome/puppet-test/hello" should not exist + When the puppet module "puppet-test" from "awesome" has a branch named "execute" + And the puppet module "puppet-test" from "awesome" has, in branch "execute", a file named "hello" with: + """ + Hello + """ + When I successfully run `msync reset --offline` + Then the file "modules/awesome/puppet-test/hello" should not exist + + Scenario: Reset to a specified branch + Given a basic setup with a puppet module "puppet-test" from "awesome" + And the global option "branch" sets to "modulesync" + When the puppet module "puppet-test" from "awesome" has a branch named "other-branch" + And the puppet module "puppet-test" from "awesome" has, in branch "other-branch", a file named "hello" with: + """ + Hello + """ + And I successfully run `msync reset` + Then the file "modules/awesome/puppet-test/hello" should not exist + When I successfully run `msync reset --verbose --source-branch origin/other-branch` + And the output should contain "Hard-resetting any local changes to repository in 'modules/awesome/puppet-test' from branch 'origin/other-branch'" + Then the file "modules/awesome/puppet-test/hello" should exist diff --git a/features/step_definitions/git_steps.rb b/features/step_definitions/git_steps.rb index ffff1372..e8848ff9 100644 --- a/features/step_definitions/git_steps.rb +++ b/features/step_definitions/git_steps.rb @@ -32,7 +32,20 @@ end Given 'a git_base option appended to "modulesync.yml" for local tests' do - File.write "#{Aruba.config.working_directory}/modulesync.yml", "\ngit_base: #{ModuleSync::Faker::PuppetModuleRemoteRepo.git_base}", mode: 'a' + step "the global option 'git_base' sets to '#{ModuleSync::Faker::PuppetModuleRemoteRepo.git_base}'" +end + +Given 'the file {string} appended with:' do |filename, content| + File.write filename, "\n#{content}", mode: 'a' +end + +Given 'the global option {string} sets to {string}' do |key, value| + steps %( + Given the file "#{Aruba.config.working_directory}/modulesync.yml" appended with: + """ + #{key}: #{value} + """ + ) end Given 'the puppet module {string} from {string} is read-only' do |name, namespace| diff --git a/features/update.feature b/features/update.feature index 702b4032..bbce2920 100644 --- a/features/update.feature +++ b/features/update.feature @@ -339,6 +339,7 @@ Feature: update Scenario: Setting a directory to unmanaged Given a basic setup with a puppet module "puppet-apache" from "puppetlabs" + And I successfully run `msync clone` And a file named "config_defaults.yml" with: """ --- @@ -395,6 +396,7 @@ Feature: update Scenario: Updating offline Given a basic setup with a puppet module "puppet-test" from "fakenamespace" + And I successfully run `msync clone` And a file named "config_defaults.yml" with: """ --- diff --git a/lib/modulesync.rb b/lib/modulesync.rb index 1aa2343c..0e9f2c24 100644 --- a/lib/modulesync.rb +++ b/lib/modulesync.rb @@ -1,3 +1,4 @@ +require 'English' require 'fileutils' require 'pathname' @@ -108,7 +109,12 @@ def self.manage_file(puppet_module, filename, settings, options) def self.manage_module(puppet_module, module_files, defaults) puts "Syncing '#{puppet_module.given_name}'" - puppet_module.repository.prepare_workspace(options[:branch]) unless options[:offline] + # NOTE: #prepare_workspace now supports to execute only offline operations + # but we totally skip the workspace preparation to keep the current behavior + unless options[:offline] + puppet_module.repository.prepare_workspace(branch: options[:branch], + operate_offline: false) + end module_configs = Util.parse_config puppet_module.path(MODULE_CONF_FILE) settings = Settings.new(defaults[GLOBAL_DEFAULTS_KEY] || {}, @@ -164,7 +170,7 @@ def self.update(cli_options) managed_modules.each do |puppet_module| manage_module(puppet_module, module_files, defaults) rescue ModuleSync::Error, Git::GitExecuteError => e - message = e.message || "Error during '#{options[:command]}'" + message = e.message || 'Error during `update`' $stderr.puts "#{puppet_module.given_name}: #{message}" exit 1 unless options[:skip_broken] errors = true @@ -177,4 +183,84 @@ def self.update(cli_options) end exit 1 if errors && options[:fail_on_warnings] end + + def self.clone(cli_options) + @options = config_defaults.merge(cli_options) + + managed_modules.each do |puppet_module| + puppet_module.repository.clone unless puppet_module.repository.cloned? + end + end + + def self.execute(cli_options) + @options = config_defaults.merge(cli_options) + + errors = {} + managed_modules.each do |puppet_module| + $stdout.puts "#{puppet_module.given_name}:" + + puppet_module.repository.clone unless puppet_module.repository.cloned? + puppet_module.repository.switch branch: @options[:branch] + + command_args = cli_options[:command_args] + local_script = File.expand_path command_args[0] + command_args[0] = local_script if File.exist?(local_script) + + # Remove bundler-related env vars to allow the subprocess to run `bundle` + command_env = ENV.reject { |k, _v| k.match?(/(^BUNDLE|^SOURCE_DATE_EPOCH$|^GEM_|RUBY)/) } + + result = system command_env, *command_args, unsetenv_others: true, chdir: puppet_module.working_directory + unless result + message = "Command execution failed ('#{@options[:command_args].join ' '}': #{$CHILD_STATUS})" + raise Thor::Error, message if @options[:fail_fast] + + errors.merge!( + puppet_module.given_name => message, + ) + $stderr.puts message + end + + $stdout.puts '' + end + + unless errors.empty? + raise Thor::Error, <<~MSG + Error(s) during `execute` command: + #{errors.map { |name, message| " * #{name}: #{message}" }.join "\n"} + MSG + end + + exit 1 unless errors.empty? + end + + def self.reset(cli_options) + @options = config_defaults.merge(cli_options) + if @options[:branch].nil? + raise Thor::Error, + "Error: 'branch' option is missing, please set it in configuration or in command line." + end + + managed_modules.each do |puppet_module| + puppet_module.repository.reset_workspace( + branch: @options[:branch], + source_branch: @options[:source_branch], + operate_offline: @options[:offline], + ) + end + end + + def self.push(cli_options) + @options = config_defaults.merge(cli_options) + + if @options[:branch].nil? + raise Thor::Error, + "Error: 'branch' option is missing, please set it in configuration or in command line." + end + + managed_modules.each do |puppet_module| + puppet_module.repository.push branch: @options[:branch], remote_branch: @options[:remote_branch] + rescue ModuleSync::Error => e + raise Thor::Error, "#{puppet_module.given_name}: #{e.message}" + end + end end diff --git a/lib/modulesync/cli.rb b/lib/modulesync/cli.rb index 0567303a..9807a924 100644 --- a/lib/modulesync/cli.rb +++ b/lib/modulesync/cli.rb @@ -7,6 +7,14 @@ module ModuleSync module CLI + def self.prepare_options(cli_options, **more_options) + options = CLI.defaults + options.merge! Util.symbolize_keys(cli_options) + options.merge! more_options + + Util.symbolize_keys options + end + def self.defaults @defaults ||= Util.symbolize_keys(Util.parse_config(Constants::MODULESYNC_CONF_FILE)) end @@ -21,16 +29,12 @@ class Hook < Thor :default => CLI.defaults[:branch] desc 'activate', 'Activate the git hook.' def activate - config = { :command => 'hook' }.merge(options) - config[:hook] = 'activate' - ModuleSync.hook(config) + ModuleSync.hook CLI.prepare_options(options, hook: 'activate') end desc 'deactivate', 'Deactivate the git hook.' def deactivate - config = { :command => 'hook' }.merge(options) - config[:hook] = 'deactivate' - ModuleSync.hook(config) + ModuleSync.hook CLI.prepare_options(options, hook: 'deactivate') end end @@ -38,7 +42,7 @@ class Base < Thor class_option :project_root, :aliases => '-c', :desc => 'Path used by git to clone modules into.', - :default => CLI.defaults[:project_root] || 'modules' + :default => CLI.defaults[:project_root] class_option :git_base, :desc => 'Specify the base part of a git URL to pull from', :default => CLI.defaults[:git_base] || 'git@github.com:' @@ -136,16 +140,101 @@ class Base < Thor :desc => 'Branch name to make the changes in.' \ ' Defaults to the default branch of the upstream repository, but falls back to "master".', :default => CLI.defaults[:branch] - def update - config = { :command => 'update' }.merge(options) - config = Util.symbolize_keys(config) + config = CLI.prepare_options(options) raise Thor::Error, 'No value provided for required option "--message"' unless config[:noop] \ || config[:message] \ || config[:offline] - config[:git_opts] = { 'amend' => config[:amend], 'force' => config[:force] } - ModuleSync.update(config) + ModuleSync.update config + end + + desc 'execute [OPTIONS] -- COMMAND..', 'Execute the command in each managed modules' + long_desc <<~DESC + Execute the command in each managed modules. + + COMMAND can be an absolute or a relative path. + + To ease running local commands, a relative path is expanded with the current user directory but only if the target file exists. + + Example: `msync exec custom-scripts/true` will run "$PWD/custom-scripts/true" in each repository. + + As side effect, you can shadow system binary if a local file is present: + \x5 `msync exec true` will run "$PWD/true", not `/bin/true` if "$PWD/true" exists. + DESC + + option :configs, + :aliases => '-c', + :desc => 'The local directory or remote repository to define the list of managed modules,' \ + ' the file templates, and the default values for template variables.' + option :managed_modules_conf, + :desc => 'The file name to define the list of managed modules' + option :branch, + :aliases => '-b', + :desc => 'Branch name to make the changes in.', + :default => CLI.defaults[:branch] + option :fail_fast, + :type => :boolean, + :desc => 'Abort the run after a command execution failure', + :default => CLI.defaults[:fail_fast].nil? ? true : CLI.defaults[:fail_fast] + def execute(*command_args) + raise Thor::Error, 'COMMAND is a required argument' if command_args.empty? + + ModuleSync.execute CLI.prepare_options(options, command_args: command_args) + end + + desc 'reset', 'Reset local repositories to a well-known state' + long_desc <<~DESC + Reset local repository to a well-known state: + \x5 * Switch local repositories to specified branch + \x5 * Fetch and prune repositories unless running with `--offline` option + \x5 * Hard-reset any changes to specified source branch, technically any git refs, e.g. `main`, `origin/wip` + \x5 * Clean all extra local files + + Note: If a repository is not already cloned, it will operate the following to reach to well-known state: + \x5 * Clone the repository + \x5 * Switch to specified branch + DESC + option :configs, + :aliases => '-c', + :desc => 'The local directory or remote repository to define the list of managed modules,' \ + ' the file templates, and the default values for template variables.' + option :managed_modules_conf, + :desc => 'The file name to define the list of managed modules' + option :branch, + :aliases => '-b', + :desc => 'Branch name to make the changes in.', + :default => CLI.defaults[:branch] + option :offline, + :type => :boolean, + :desc => 'Only proceed local operations', + :default => false + option :source_branch, + :desc => 'Branch to reset from (e.g. origin/wip)' + def reset + ModuleSync.reset CLI.prepare_options(options) + end + + desc 'push', 'Push all available commits from branch to remote' + option :configs, + :aliases => '-c', + :desc => 'The local directory or remote repository to define the list of managed modules,' \ + ' the file templates, and the default values for template variables.' + option :managed_modules_conf, + :desc => 'The file name to define the list of managed modules' + option :branch, + :aliases => '-b', + :desc => 'Branch name to push', + :default => CLI.defaults[:branch] + option :remote_branch, + :desc => 'Remote branch to push to (e.g. maintenance)' + def push + ModuleSync.push CLI.prepare_options(options) + end + + desc 'clone', 'Clone repositories that need to' + def clone + ModuleSync.clone CLI.prepare_options(options) end desc 'hook', 'Activate or deactivate a git hook.' diff --git a/lib/modulesync/hook.rb b/lib/modulesync/hook.rb index 1be023cc..81d52a7b 100644 --- a/lib/modulesync/hook.rb +++ b/lib/modulesync/hook.rb @@ -6,9 +6,9 @@ class Hook def initialize(hook_file, options = []) @hook_file = hook_file - @namespace = options['namespace'] - @branch = options['branch'] - @args = options['hook_args'] + @namespace = options[:namespace] + @branch = options[:branch] + @args = options[:hook_args] end def content(arguments) @@ -28,9 +28,7 @@ def activate hook_args << "-b #{branch}" if branch hook_args << args if args - File.open(hook_file, 'w') do |file| - file.write(content(hook_args.join(' '))) - end + File.write(hook_file, content(hook_args.join(' '))) end def deactivate diff --git a/lib/modulesync/renderer.rb b/lib/modulesync/renderer.rb index 188f9f9e..dfe295c6 100644 --- a/lib/modulesync/renderer.rb +++ b/lib/modulesync/renderer.rb @@ -34,9 +34,7 @@ def self.render(_template, configs = {}, metadata = {}) def self.sync(template, target_name) path = target_name.rpartition('/').first FileUtils.mkdir_p(path) unless path.empty? - File.open(target_name, 'w') do |file| - file.write(template) - end + File.write(target_name, template) end end end diff --git a/lib/modulesync/repository.rb b/lib/modulesync/repository.rb index 58582b29..91a7c4bd 100644 --- a/lib/modulesync/repository.rb +++ b/lib/modulesync/repository.rb @@ -37,7 +37,7 @@ def default_branch %r{remotes/origin/HEAD\s+->\s+origin/(?.+?)$}.match(symbolic_ref.full)[:branch] end - def switch_branch(branch) + def switch(branch:) unless branch branch = default_branch puts "Using repository's default branch: #{branch}" @@ -63,23 +63,50 @@ def switch_branch(branch) end end - def prepare_workspace(branch) - # Repo already cloned, check out master and override local changes - if Dir.exist? File.join(@directory, '.git') - # Some versions of git can't properly handle managing a repo from outside the repo directory - Dir.chdir(@directory) do - puts "Overriding any local changes to repository in '#{@directory}'" - @git = Git.open('.') - repo.fetch 'origin', prune: true - repo.reset_hard - switch_branch(branch) - git.pull('origin', branch) if remote_branch_exists?(branch) - end - # Repo needs to be cloned in the cwd + def cloned? + Dir.exist? File.join(@directory, '.git') + end + + def clone + puts "Cloning from '#{@remote}'" + @git = Git.clone(@remote, @directory) + end + + def prepare_workspace(branch:, operate_offline:) + if cloned? + puts "Overriding any local changes to repository in '#{@directory}'" + git.fetch 'origin', prune: true unless operate_offline + git.reset_hard + switch(branch: branch) + git.pull('origin', branch) if !operate_offline && remote_branch_exists?(branch) else - puts "Cloning from '#{@remote}'" - @git = Git.clone(@remote, @directory) - switch_branch(branch) + raise ModuleSync::Error, 'Unable to clone in offline mode.' if operate_offline + + clone + switch(branch: branch) + end + end + + def default_reset_branch(branch) + remote_branch_exists?(branch) ? branch : default_branch + end + + def reset_workspace(branch:, operate_offline:, source_branch: nil) + raise if branch.nil? + + if cloned? + source_branch ||= "origin/#{default_reset_branch branch}" + puts "Hard-resetting any local changes to repository in '#{@directory}' from branch '#{source_branch}'" + switch(branch: branch) + git.fetch 'origin', prune: true unless operate_offline + + git.reset_hard source_branch + git.clean(d: true, force: true) + else + raise ModuleSync::Error, 'Unable to clone in offline mode.' if operate_offline + + clone + switch(branch: branch) end end @@ -136,6 +163,16 @@ def submit_changes(files, options) true end + def push(branch:, remote_branch:, remote_name: 'origin') + raise ModuleSync::Error, 'Repository must be locally available before trying to push' unless cloned? + + remote_url = git.remote(remote_name).url + remote_branch ||= branch + puts "Push branch '#{branch}' to '#{remote_url}' (#{remote_name}/#{remote_branch})" + + git.push(remote_name, "#{branch}:#{remote_branch}", force: true) + end + # Needed because of a bug in the git gem that lists ignored files as # untracked under some circumstances # https://github.com/schacon/ruby-git/issues/130