From 4dd912a00ae3f4d9ecc9ed013168cc6f7f9dd708 Mon Sep 17 00:00:00 2001 From: Tony Hsu Date: Wed, 18 Sep 2024 11:44:09 +0200 Subject: [PATCH] Improve update gemfiles --- .github/workflows/lock-dependency.yml | 93 +++++++++++++++++++++++++++ .github/workflows/update-gemfiles.yml | 83 ------------------------ Appraisals | 2 +- appraisal/jruby-9.2.rb | 2 +- appraisal/jruby-9.3.rb | 2 +- appraisal/jruby-9.4.rb | 2 +- appraisal/ruby-2.5.rb | 2 +- appraisal/ruby-2.6.rb | 2 +- appraisal/ruby-2.7.rb | 2 +- appraisal/ruby-3.0.rb | 2 +- appraisal/ruby-3.1.rb | 2 +- appraisal/ruby-3.2.rb | 2 +- appraisal/ruby-3.3.rb | 2 +- appraisal/ruby-3.4.rb | 2 +- tasks/appraisal_conversion.rb | 49 ++++++++++++++ tasks/dependency.rake | 49 ++++++++++++++ tasks/edge.rake | 43 +------------ 17 files changed, 204 insertions(+), 137 deletions(-) create mode 100644 .github/workflows/lock-dependency.yml delete mode 100644 .github/workflows/update-gemfiles.yml create mode 100644 tasks/appraisal_conversion.rb create mode 100644 tasks/dependency.rake diff --git a/.github/workflows/lock-dependency.yml b/.github/workflows/lock-dependency.yml new file mode 100644 index 00000000000..1b977e1f241 --- /dev/null +++ b/.github/workflows/lock-dependency.yml @@ -0,0 +1,93 @@ +name: Lock Dependency + +on: + workflow_dispatch: + inputs: + branch: + description: 'Branch to be lock dependency' + required: true + # Testing purpose, to be removed before merge. + push: + branches: + - tonycthsu/automate-update-gemfiles + +# Ensure obsolete job is cancelled if another commit is pushed to the same branch. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + # TODO: In order to fully automate this workflow for each PR, + # have a reliable way to precheck job to understand whether it need to be updated + lock: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + engine: + # ADD NEW RUBIES HERE + - name: ruby + version: '3.4' + - name: ruby + version: '3.3' + - name: ruby + version: '3.2' + - name: ruby + version: '3.1' + - name: ruby + version: '3.0' + - name: ruby + version: '2.7' + - name: ruby + version: '2.6' + - name: ruby + version: '2.5' + - name: jruby + version: '9.4' + - name: jruby + version: '9.3' + - name: jruby + version: '9.2' + container: + image: ghcr.io/datadog/images-rb/engines/${{ matrix.engine.name }}:${{ matrix.engine.version }} + env: + BUNDLE_WITHOUT: check + steps: + - uses: actions/checkout@v4 + - run: ls -al + - run: | + ruby -v + gem -v + bundler -v + + - run: bundle install + + # TODO: Migrate away from `appraisal` + - run: bundle exec appraisal generate + - run: bundle exec rake dependency:lock + + - uses: actions/upload-artifact@v4 + with: + name: lock-dependency-${{ github.run_id }}-${{ matrix.engine.name }}-${{ matrix.engine.version }} + path: gemfiles/${{ matrix.engine.name }}_${{ matrix.engine.version }}* + + commit: + needs: lock + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + + - uses: actions/download-artifact@v4 + with: + path: gemfiles + pattern: lock-dependency-${{ github.run_id }}-* + merge-multiple: true + + - run: git diff --color + + - uses: stefanzweifel/git-auto-commit-action@v5 + with: + file_pattern: 'gemfiles/*' + commit_message: "[🤖] Lock Dependency: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" diff --git a/.github/workflows/update-gemfiles.yml b/.github/workflows/update-gemfiles.yml deleted file mode 100644 index 078a15ccd3d..00000000000 --- a/.github/workflows/update-gemfiles.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: Update Gemfiles - -# This action cannot be skipped altogether because it is a mandatory status check. -# Instead we conditionally skip it at job level, instead of workflow level. -on: - # Execute on `push` and not `pull_request` because `pull_request` - # always compares if the `paths` have changed compared to the PR base. - # This is an issue because it means that all commits to the branch - # will trigger the gemfile update process, which is unnecessary and expensive. - # - # By executing on `push`, GitHub compares `paths` with the parent commit, - # meaning the gemfile update process will only execute on the exact commit - # that changes any of the `paths`. - # - # Because this process is slow and expensive, and we commit the gemfile changes back - # to the branch, we have an additional filter to only execute this action on branches - # attached to a PR. - # - # We could do the inverse: execute this action on `pull_request`, and additionally check - # if `paths` was changed compared to the parent commit, but this proved more complicated. - push - -# Ensure obsolete job is cancelled if another commit is pushed to the same branch. -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - check: - name: Update Gemfiles - runs-on: ubuntu-22.04 - permissions: - contents: write - pull-requests: write - steps: - # Only execute if there's a PR attached to this branch. - # Because we execute on `push`, we have to double check here if this is part of a PR. - - name: Check if this branch is attached to a Pull Request - uses: 8bitjonny/gh-get-current-pr@2215326c76d51bfa3f2af0a470f32677f6c0cae9 # v2.2.0 - id: pr - with: - filterOutClosed: true # Don't trigger on commits with closed PRs, including merges into `master`. - - if: steps.pr.outputs.pr_found == 'true' - uses: actions/checkout@v4 - # And also, only execute if files that can affect gemfiles are modified. - - if: steps.pr.outputs.pr_found == 'true' - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 - id: filter - with: - base: ${{ github.ref_name }} - filters: | - gemfile: - # Files that declare the dependency tree - - Gemfile - - Appraisals - - datadog.gemspec - # Files that control gemfile generation - - tasks/appraisal.rake - - .github/workflows/update-gemfiles.yml - # The gem version is present in all lock files - - lib/datadog/version.rb - # In case the generated files were updated manually or in a merge commit - - appraisal/** - - gemfiles/** - - if: steps.pr.outputs.pr_found == 'true' && steps.filter.outputs.gemfile == 'true' - uses: ruby/setup-ruby@v1 - with: - ruby-version: '3.2' - bundler-cache: true # runs 'bundle install' and caches installed gems automatically - - if: steps.pr.outputs.pr_found == 'true' && steps.filter.outputs.gemfile == 'true' - name: Ensure gemfiles/*.gemfile.lock match gem definition - run: bundle exec rake appraisal:lock - - if: steps.pr.outputs.pr_found == 'true' && steps.filter.outputs.gemfile == 'true' - name: Add all supported platforms to gemfiles/*.gemfile.lock - run: bundle exec rake appraisal:platform - - if: steps.pr.outputs.pr_found == 'true' && steps.filter.outputs.gemfile == 'true' - name: Remove obsolete gemfiles/* - run: bundle exec rake appraisal:clean - - if: steps.pr.outputs.pr_found == 'true' && steps.filter.outputs.gemfile == 'true' - name: Commit gemfiles changes, if any, back to the branch - uses: stefanzweifel/git-auto-commit-action@3ea6ae190baf489ba007f7c92608f33ce20ef04a # v4.16.0 - with: - commit_message: Update gemfiles/* diff --git a/Appraisals b/Appraisals index b312f409b3e..ef649f43103 100644 --- a/Appraisals +++ b/Appraisals @@ -52,7 +52,7 @@ def build_coverage_matrix(integration, range, gem: nil, min: nil, meta: {}) if min appraise "#{integration}-min" do - gem gem, "= #{n}" + gem gem, "= #{min}" meta.each { |k, v| gem k, v } end end diff --git a/appraisal/jruby-9.2.rb b/appraisal/jruby-9.2.rb index cd3a5619f49..ed8c523b69f 100644 --- a/appraisal/jruby-9.2.rb +++ b/appraisal/jruby-9.2.rb @@ -189,7 +189,7 @@ gem 'typhoeus' end -build_coverage_matrix('stripe', 7..12) +build_coverage_matrix('stripe', 7..12, min: '5.15.0') build_coverage_matrix('opensearch', 2..3, gem: 'opensearch-ruby') build_coverage_matrix('elasticsearch', 7..8) diff --git a/appraisal/jruby-9.3.rb b/appraisal/jruby-9.3.rb index 3a98ded6920..6556d4f81bb 100644 --- a/appraisal/jruby-9.3.rb +++ b/appraisal/jruby-9.3.rb @@ -162,7 +162,7 @@ gem 'typhoeus' end -build_coverage_matrix('stripe', 7..12) +build_coverage_matrix('stripe', 7..12, min: '5.15.0') build_coverage_matrix('opensearch', 2..3, gem: 'opensearch-ruby') build_coverage_matrix('elasticsearch', 7..8) diff --git a/appraisal/jruby-9.4.rb b/appraisal/jruby-9.4.rb index 1968306e239..2bf7c0d5698 100644 --- a/appraisal/jruby-9.4.rb +++ b/appraisal/jruby-9.4.rb @@ -66,7 +66,7 @@ gem 'typhoeus' end -build_coverage_matrix('stripe', 7..12) +build_coverage_matrix('stripe', 7..12, min: '5.15.0') build_coverage_matrix('opensearch', 2..3, gem: 'opensearch-ruby') build_coverage_matrix('elasticsearch', 7..8) diff --git a/appraisal/ruby-2.5.rb b/appraisal/ruby-2.5.rb index 2e8d2cf9772..d0ff148a8d1 100644 --- a/appraisal/ruby-2.5.rb +++ b/appraisal/ruby-2.5.rb @@ -209,7 +209,7 @@ gem 'typhoeus' end -build_coverage_matrix('stripe', 7..12) +build_coverage_matrix('stripe', 7..12, min: '5.15.0') build_coverage_matrix('opensearch', 2..3, gem: 'opensearch-ruby') build_coverage_matrix('elasticsearch', 7..8) diff --git a/appraisal/ruby-2.6.rb b/appraisal/ruby-2.6.rb index 312a76cdba9..45d196a33b0 100644 --- a/appraisal/ruby-2.6.rb +++ b/appraisal/ruby-2.6.rb @@ -162,7 +162,7 @@ gem 'typhoeus' end -build_coverage_matrix('stripe', 7..12) +build_coverage_matrix('stripe', 7..12, min: '5.15.0') build_coverage_matrix('opensearch', 2..3, gem: 'opensearch-ruby') build_coverage_matrix('elasticsearch', 7..8) diff --git a/appraisal/ruby-2.7.rb b/appraisal/ruby-2.7.rb index 4e02ac9adc2..e7c203513b7 100644 --- a/appraisal/ruby-2.7.rb +++ b/appraisal/ruby-2.7.rb @@ -162,7 +162,7 @@ gem 'typhoeus' end -build_coverage_matrix('stripe', 7..12) +build_coverage_matrix('stripe', 7..12, min: '5.15.0') build_coverage_matrix('opensearch', 2..3, gem: 'opensearch-ruby') build_coverage_matrix('elasticsearch', 7..8) diff --git a/appraisal/ruby-3.0.rb b/appraisal/ruby-3.0.rb index 7dcd32e7a3f..c0f2d58b3fc 100644 --- a/appraisal/ruby-3.0.rb +++ b/appraisal/ruby-3.0.rb @@ -75,7 +75,7 @@ gem 'typhoeus' end -build_coverage_matrix('stripe', 7..12) +build_coverage_matrix('stripe', 7..12, min: '5.15.0') build_coverage_matrix('opensearch', 2..3, gem: 'opensearch-ruby') build_coverage_matrix('elasticsearch', 7..8) diff --git a/appraisal/ruby-3.1.rb b/appraisal/ruby-3.1.rb index 7dcd32e7a3f..c0f2d58b3fc 100644 --- a/appraisal/ruby-3.1.rb +++ b/appraisal/ruby-3.1.rb @@ -75,7 +75,7 @@ gem 'typhoeus' end -build_coverage_matrix('stripe', 7..12) +build_coverage_matrix('stripe', 7..12, min: '5.15.0') build_coverage_matrix('opensearch', 2..3, gem: 'opensearch-ruby') build_coverage_matrix('elasticsearch', 7..8) diff --git a/appraisal/ruby-3.2.rb b/appraisal/ruby-3.2.rb index 7dcd32e7a3f..c0f2d58b3fc 100644 --- a/appraisal/ruby-3.2.rb +++ b/appraisal/ruby-3.2.rb @@ -75,7 +75,7 @@ gem 'typhoeus' end -build_coverage_matrix('stripe', 7..12) +build_coverage_matrix('stripe', 7..12, min: '5.15.0') build_coverage_matrix('opensearch', 2..3, gem: 'opensearch-ruby') build_coverage_matrix('elasticsearch', 7..8) diff --git a/appraisal/ruby-3.3.rb b/appraisal/ruby-3.3.rb index 6bdd3958304..c740488fb1a 100644 --- a/appraisal/ruby-3.3.rb +++ b/appraisal/ruby-3.3.rb @@ -75,7 +75,7 @@ gem 'typhoeus' end -build_coverage_matrix('stripe', 7..12) +build_coverage_matrix('stripe', 7..12, min: '5.15.0') build_coverage_matrix('opensearch', 2..3, gem: 'opensearch-ruby') build_coverage_matrix('elasticsearch', 7..8) diff --git a/appraisal/ruby-3.4.rb b/appraisal/ruby-3.4.rb index e38331b9dba..97baa215598 100644 --- a/appraisal/ruby-3.4.rb +++ b/appraisal/ruby-3.4.rb @@ -75,7 +75,7 @@ gem 'typhoeus' end -build_coverage_matrix('stripe', 7..12) +build_coverage_matrix('stripe', 7..12, min: '5.15.0') build_coverage_matrix('opensearch', 2..3, gem: 'opensearch-ruby') build_coverage_matrix('elasticsearch', 7..8) diff --git a/tasks/appraisal_conversion.rb b/tasks/appraisal_conversion.rb new file mode 100644 index 00000000000..685c544911f --- /dev/null +++ b/tasks/appraisal_conversion.rb @@ -0,0 +1,49 @@ +require 'pathname' + +# This module translates our custom mapping between appraisal and bundler. +# +# It cannot be included into `Appraisal` file, because it was invoked via `instance_eval`. +module AppraisalConversion + module_function + + @gemfile_dir = 'gemfiles' + @definition_dir = 'appraisal' + + def to_bundle_gemfile(group) + gemfile = "#{runtime_identifier}_#{group}.gemfile".tr('-', '_') + path = root_path.join(gemfile_dir, gemfile) + + if path.exist? + path.to_s + else + raise "Gemfile not found at #{path}" + end + end + + def definition + path = root_path.join(@definition_dir, "#{runtime_identifier}.rb") + + if path.exist? + path.to_s + else + raise "Definition not found at #{path}" + end + end + + def runtime_identifier + major, minor, = Gem::Version.new(RUBY_ENGINE_VERSION).segments + "#{RUBY_ENGINE}-#{major}.#{minor}" + end + + def gemfile_pattern + root_path + gemfile_dir + "#{runtime_identifier.tr('-', '_')}_*.gemfile" + end + + def gemfile_dir + @gemfile_dir + end + + def root_path + Pathname.pwd + end +end diff --git a/tasks/dependency.rake b/tasks/dependency.rake new file mode 100644 index 00000000000..4af85eb1e9f --- /dev/null +++ b/tasks/dependency.rake @@ -0,0 +1,49 @@ +require 'open3' + +require_relative 'appraisal_conversion' + +task :dep => :dependency +task :dependency => %w[dependency:lock] +namespace :dependency do + # rubocop:disable Style/MultilineBlockChain + Dir.glob(AppraisalConversion.gemfile_pattern).each do |gemfile| + # desc "Lock the dependencies for #{gemfile}" + task gemfile do + Bundler.with_unbundled_env do + command = +'bundle lock' + command << ' --add-platform x86_64-linux aarch64-linux' unless RUBY_PLATFORM == 'java' + output, = Open3.capture2e({ 'BUNDLE_GEMFILE' => gemfile.to_s }, command) + + puts output + end + end + end.tap do |gemfiles| + desc "Lock the dependencies for #{AppraisalConversion.runtime_identifier}" + # WHY can't we use `multitask :lock => gemfiles` here? + # + # Running bundler in parallel has various race conditions + # + # Race condition with the file system, particularly worse with JRuby. + # For instance, `Errno::ENOENT: No such file or directory - bundle` is raised with JRuby 9.2 + + # Even with CRuby, `simplcov` declaration with `github` in Gemfile causes + # race condition for the local gem cache with the following error: + + # ``` + # [/usr/local/bundle/bundler/gems/simplecov-3bb6b7ee58bf/simplecov.gemspec] isn't a Gem::Specification (NilClass instead). + # ``` + + # and + + # ``` + # fatal: Unable to create '/usr/local/bundle/bundler/gems/simplecov-3bb6b7ee58bf/.git/index.lock': File exists. + # Another git process seems to be running in this repository, e.g. + # an editor opened by 'git commit'. Please make sure all processes + # are terminated then try again. If it still fails, a git process + # may have crashed in this repository earlier: + # remove the file manually to continue. + # ``` + task :lock => gemfiles + end + # rubocop:enable Style/MultilineBlockChain +end diff --git a/tasks/edge.rake b/tasks/edge.rake index aee5e675150..5ee9766ccff 100644 --- a/tasks/edge.rake +++ b/tasks/edge.rake @@ -1,47 +1,6 @@ -# frozen_string_literal: true - require 'open3' -require 'pathname' - -# This module translates our custom mapping between appraisal and bundler. -# -# It cannot be included into `Appraisal` file, because it was invoked via `instance_eval`. -module AppraisalConversion - module_function - - @gemfile_dir = 'gemfiles' - @definition_dir = 'appraisal' - - def to_bundle_gemfile(group) - gemfile = "#{runtime_identifier}_#{group}.gemfile".tr('-', '_') - path = root_path.join(@gemfile_dir, gemfile) - - if path.exist? - path.to_s - else - raise "Gemfile not found at #{path}" - end - end - - def definition - path = root_path.join(@definition_dir, "#{runtime_identifier}.rb") - if path.exist? - path.to_s - else - raise "Definition not found at #{path}" - end - end - - def runtime_identifier - major, minor, = Gem::Version.new(RUBY_ENGINE_VERSION).segments - "#{RUBY_ENGINE}-#{major}.#{minor}" - end - - def root_path - Pathname.pwd - end -end +require_relative 'appraisal_conversion' # rubocop:disable Metrics/BlockLength namespace :edge do