feat: switch to using package_json for interacting with `package.js…

…on` (#466)

This allows us to support all of the major javascript package managers
by relying on the
[`package_json`](https://github.com/shakacode/package_json) gem to
handle running and generating commands for a particular package manager.

Currently I'm making our template actually agnostic and have setup to
test that is the case by running against the major package managers +
Yarn PnP, though I expect after landing this we'll decide on a single
package manager to use going forward and remove code that is needed for
the other package managers.

.github/workflows/ci.yml

@@ -18,6 +18,13 @@ concurrency:
 # should be granted per job as needed using a dedicated `permissions` block
 permissions: {}
 
+env:
+  # reduces noise from npm install
+  DISABLE_OPENCOLLECTIVE: true
+  OPEN_SOURCE_CONTRIBUTOR: true
+  NPM_CONFIG_FUND: false
+  NPM_CONFIG_AUDIT: false
+
 jobs:
   audit_dependencies:
     permissions:
@@ -29,6 +36,8 @@ jobs:
           persist-credentials: false
       - name: Audit dependencies for security vulnerabilities
         uses: g-rath/check-with-osv-detector@main
+        with:
+          osv-detector-version: 0.13.0
   test:
     permissions:
       contents: read
@@ -70,6 +79,24 @@ jobs:
       # how many fail)
       fail-fast: false
       matrix:
+        js_package_manager:
+          - name: npm
+            installer: npm
+          - name: yarn_berry
+            installer: yarn
+            linker: pnp
+          - name: yarn_berry
+            installer: yarn
+            linker: node-modules
+          - name: yarn_berry
+            installer: yarn
+            linker: pnpm
+          - name: yarn_classic
+            installer: yarn
+          - name: pnpm
+            installer: pnpm
+          - name: bun
+            installer: bun
         variant:
           - name: defaults
             config_path: 'ackama_rails_template.config.yml'
@@ -131,6 +158,8 @@ jobs:
       # this ensures that osv-detector is available for running bin/ci-run
       - name: Check dependencies for vulnerabilities (and setup osv-detector)
         uses: g-rath/check-with-osv-detector@main
+        with:
+          osv-detector-version: 0.13.0
 
       # this ensures that actionlint is available for running bin/ci-run
       - name: Setup ActionLint
@@ -142,9 +171,11 @@ jobs:
       - name: Install NodeJS
         uses: actions/setup-node@v4
         with:
-          cache: 'yarn'
           node-version-file: '.node-version'
 
+      - name: install package manager
+        run: npm i -g ${{ matrix.js_package_manager.installer }}
+
       # We don't cache gems or JS packages because we are actually testing how
       # installation and setup works in this project so, while caching would
       # make CI faster, it might hide problems.
@@ -161,6 +192,9 @@ jobs:
           git config --global user.email "you@example.com"
           git config --global user.name "Your Name"
 
+      # prettier-ignore
+      - run: ./ci/bin/create-fake-js-package-managers ${{ matrix.js_package_manager.installer }}
+
       - name: Run CI script
         env:
           # Remember that your app name becomes a top-level constant in the
@@ -173,4 +207,7 @@ jobs:
           PGUSER: postgres
           PGPASSWORD: postgres
           PGHOST: localhost
+          PACKAGE_JSON_FALLBACK_MANAGER: ${{ matrix.js_package_manager.name }}
+          PACKAGE_JSON_YARN_BERRY_LINKER:
+            ${{ matrix.js_package_manager.linker }}
         run: ./ci/bin/build-and-test

ci/bin/create-fake-js-package-managers

@@ -0,0 +1,62 @@
+#!/usr/bin/env ruby
+
+# creates a set of fake JavaScript package managers in a temporary bin
+# directory for GitHub Actions, _excluding_ the one passed in as an
+# argument in order to assert that only that package manager is used
+
+require "fileutils"
+require "tmpdir"
+
+# setup the bin directory we want to use
+bin_dir = "tmp/fake-bin"
+
+if ENV["GITHUB_ACTIONS"]
+  bin_dir = Dir.mktmpdir("rails-template-")
+
+  puts "adding #{bin_dir} to GITHUB_PATH..."
+
+  File.write(ENV.fetch("GITHUB_PATH"), "#{bin_dir}\n", mode: "a+")
+elsif system("direnv --version > /dev/null 2>&1")
+  envrc_content = "PATH_add #{bin_dir}\n"
+
+  if File.exist?(".envrc") && File.read(".envrc").include?(envrc_content)
+    puts "'#{envrc_content.strip}' already exists in .envrc"
+  else
+    File.write(".envrc", envrc_content, mode: "a")
+    puts "Added '#{envrc_content.strip}' to .envrc"
+  end
+
+  # ensure the .envrc is allowed
+  system("direnv allow")
+end
+
+managers = %w[npm yarn pnpm bun]
+manager_in_use = ARGV[0] || ""
+
+if manager_in_use.empty?
+  manager_in_use = ENV.fetch("PACKAGE_JSON_FALLBACK_MANAGER", "")
+                      .delete_suffix("_berry")
+                      .delete_suffix("_classic")
+end
+
+Dir.chdir(bin_dir) do
+  managers.each do |manager|
+    if manager == manager_in_use
+      # ensure that the manager is not stubbed in case we've changed managers
+      FileUtils.rm_f(manager)
+
+      next
+    end
+
+    puts "creating #{bin_dir}/#{manager}..."
+    File.write(
+      manager,
+      <<~CONTENTS
+        #!/usr/bin/env node
+
+        throw new Error("(#{manager}) this is not the package manager you're looking for...");
+      CONTENTS
+    )
+    File.chmod(0o755, manager)
+  end
+end

template.rb

@@ -92,6 +92,21 @@ def puts_header(msg)
 TEMPLATE_CONFIG = Config.new
 TERMINAL = Terminal.new
 
+# We need the major version of 'jest', '@types/jest', and 'ts-jest' to match
+# so we can only upgrade jest when there are compatible versions available
+JEST_MAJOR_VERSION = "29".freeze
+
+def require_package_json_gem
+  require "bundler/inline"
+
+  gemfile(true) do
+    source "https://rubygems.org"
+    gem "package_json"
+  end
+
+  puts "using package_json v#{PackageJson::VERSION}"
+end
+
 def apply_template! # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
   assert_minimum_rails_version
   assert_valid_options
@@ -121,14 +136,19 @@ def apply_template! # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Met
   apply "variants/backend-base/bin/template.rb"
   apply "variants/backend-base/config/template.rb"
   apply "variants/backend-base/doc/template.rb"
-  apply "variants/backend-base/lib/template.rb"
   apply "variants/backend-base/public/template.rb"
   apply "variants/backend-base/spec/template.rb"
 
   # The block passed to "after_bundle" seems to run after `bundle install`
   # but also after `shakapacker:install` and after Rails has initialized the git
   # repo
   after_bundle do # rubocop:disable Metrics/BlockLength
+    require_package_json_gem
+
+    apply "variants/backend-base/lib/template.rb"
+
+    template "variants/backend-base/bin/setup.tt", "bin/setup", force: true
+
     # Remove the `test/` directory because we always use RSpec which creates
     # its own `spec/` directory
     remove_dir "test"
@@ -150,11 +170,11 @@ def apply_template! # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Met
       apply "variants/frontend-bootstrap-typescript/template.rb" if TEMPLATE_CONFIG.apply_variant_bootstrap?
       apply "variants/frontend-react-typescript/template.rb" if TEMPLATE_CONFIG.apply_variant_react?
 
-      run "yarn run typecheck"
+      package_json.manager.run!("typecheck")
     end
 
     # apply any js linting fixes after all frontend variants have run
-    run "yarn run js-lint-fix"
+    package_json.manager.run!("js-lint-fix")
 
     create_initial_migration
 
@@ -235,11 +255,75 @@ def apply_readme_template
 end
 
 def apply_prettier_all_over
-  run "yarn run format-fix"
+  package_json.manager.run!("format-fix")
 
   git commit: ". -m 'Run prettier one last time'"
 end
 
+# Sets Yarn Berry up in the project directory by initializing it with what is probably Yarn Classic.
+#
+# This is required as the Berry binary is actually downloaded and committed to the codebase, and
+# the global yarn command passes through to it when detected (even if its Yarn Classic).
+#
+# This also requires us to temporarily create a package.json as otherwise Yarn Berry will
+# look up the file tree and initialize itself in every directory that has a yarn.lock
+def setup_yarn_berry
+  # safeguard against parent directories having a yarn.lock
+  File.write("package.json", "{}") unless File.exist?("package.json")
+
+  run "yarn init -2"
+  run "yarn config set enableGlobalCache true"
+  run "yarn config set nodeLinker #{ENV.fetch("PACKAGE_JSON_YARN_BERRY_LINKER", "node-modules")}"
+
+  ignores = <<~YARN
+    .pnp.*
+    .yarn/*
+    !.yarn/patches
+    !.yarn/plugins
+    !.yarn/releases
+    !.yarn/sdks
+    !.yarn/versions
+  YARN
+  File.write(".gitignore", ignores, mode: "a")
+
+  # this will be properly (re)created later
+  File.unlink("package.json")
+end
+
+# Bun uses a binary-based lockfile which cannot be parsed by shakapacker or
+# osv-detector, so we want to configure bun to always write a yarn.lock
+# in addition so that such tools can check it
+def setup_bun
+  File.write("bunfig.toml", <<~TOML)
+    [install.lockfile]
+    print = "yarn"
+  TOML
+end
+
+def add_yarn_package_extension_dependency(name, dependency)
+  return unless File.exist?(".yarnrc.yml")
+
+  require "yaml"
+
+  yarnrc = YAML.load_file(".yarnrc.yml")
+
+  yarnrc["packageExtensions"] ||= {}
+  yarnrc["packageExtensions"]["#{name}@*"] ||= {}
+  yarnrc["packageExtensions"]["#{name}@*"]["dependencies"] ||= {}
+  yarnrc["packageExtensions"]["#{name}@*"]["dependencies"][dependency] = "*"
+
+  File.write(".yarnrc.yml", yarnrc.to_yaml)
+end
+
+def package_json
+  if @package_json.nil?
+    setup_yarn_berry if ENV.fetch("PACKAGE_JSON_FALLBACK_MANAGER", nil) == "yarn_berry"
+    setup_bun if ENV.fetch("PACKAGE_JSON_FALLBACK_MANAGER", nil) == "bun"
+  end
+
+  @package_json ||= PackageJson.new
+end
+
 # Normalizes the constraints of the given hash of dependencies so that they
 # all have an explicit constraint and define a minor & patch version
 #
@@ -254,50 +338,48 @@ def normalize_dependency_constraints(deps)
   end
 end
 
-def build_engines_field
+def build_engines_field(existing)
   node_version = File.read("./.node-version").strip
-  {
-    node: "^#{node_version}",
-    yarn: "^1.0.0"
-  }
-end
-
-def update_package_json(&)
-  package_json = JSON.load_file("./package.json").tap(&)
 
-  File.write("./package.json", "#{JSON.pretty_generate(package_json)}\n")
+  existing.merge({
+    "node" => "^#{node_version}",
+    "yarn" => "^1.0.0"
+  })
 end
 
 def cleanup_package_json
-  update_package_json do |package_json|
-    # ensure that the package name is set based on the folder
-    package_json["name"] = File.basename(__dir__)
-
-    # set engines constraint in package.json
-    package_json["engines"] = build_engines_field
-
-    # ensure that all dependency constraints are normalized
-    %w[dependencies devDependencies].each { |k| package_json[k] = normalize_dependency_constraints(package_json[k]) }
+  package_json.merge! do |pj|
+    {
+      "name" => File.basename(__dir__),
+      "engines" => build_engines_field(pj.fetch("engines", {})),
+      "dependencies" => normalize_dependency_constraints(pj.fetch("dependencies", {})),
+      "devDependencies" => normalize_dependency_constraints(pj.fetch("devDependencies", {}))
+    }
   end
 
-  run "npx -y sort-package-json"
+  # TODO: this doesn't work when using pnpm even though it shouldn't matter? anyway, replace with 'exec' support
+  # run "npx -y sort-package-json"
 
-  # ensure the yarn.lock is up to date with any changes we've made to package.json
-  run "yarn install"
+  # ensure the lockfile is up to date with any changes we've made to package.json
+  package_json.manager.install!
 end
 
 # Adds the given <code>packages</code> as dependencies using <code>yarn add</code>
 #
 # @param [Array<String>] packages
 def yarn_add_dependencies(packages)
-  run "yarn add #{packages.join " "}"
+  puts "adding #{packages.join(" ")} as dependencies"
+
+  package_json.manager.add!(packages)
 end
 
 # Adds the given <code>packages</code> as devDependencies using <code>yarn add --dev</code>
 #
 # @param [Array<String>] packages
 def yarn_add_dev_dependencies(packages)
-  run "yarn add --dev #{packages.join " "}"
+  puts "adding #{packages.join(" ")} as dev dependencies"
+
+  package_json.manager.add!(packages, type: :dev)
 end
 
 # Add this template directory to source_paths so that Thor actions like

variants/accessibility/spec/rails_helper.rb

@@ -9,6 +9,14 @@
   <<~OPTIONS
     # Lighthouse Matcher options
     options.add_argument("--remote-debugging-port=9222")
-    Lighthouse::Matchers.chrome_flags = %w[headless no-sandbox]
+    Lighthouse::Matchers.chrome_flags = %w[headless=new no-sandbox]
   OPTIONS
 end
+
+if File.exist?(".yarnrc.yml")
+  insert_into_file "spec/rails_helper.rb", after: /# Lighthouse Matcher options\n/ do
+    <<~OPTIONS
+      Lighthouse::Matchers.lighthouse_cli = "yarn run lighthouse"
+    OPTIONS
+  end
+end

variants/backend-base/bin/setup → variants/backend-base/bin/setup.tt

@@ -5,7 +5,7 @@ def setup!
     test "ruby -v" => ruby_version
     run  "gem install bundler --no-document --conservative"
     run  "bundle install"
-    run  "yarn install" if File.exist?("yarn.lock")
+    run  "<%= package_json.manager.native_install_command.join(" ") %>" if File.exist?("package.json")
     run  "bundle exec overcommit --install" unless ENV["SKIP_OVERCOMMIT"] || ENV["CI"]
     copy "example.env"
     test_local_env_contains_required_keys

variants/backend-base/lib/tasks/assets.rake → variants/backend-base/lib/tasks/assets.rake.tt

@@ -1,7 +1,7 @@
 namespace :assets do
   desc "Ensures that dependencies required to compile assets are installed"
   task install_dependencies: :environment do
-    raise if File.exist?("yarn.lock") && !(system "yarn install --frozen-lockfile")
+    raise if File.exist?("package.json") && !(system "<%= package_json.manager.native_install_command(frozen: true).join(" ") %>")
   end
 end
 

variants/backend-base/lib/template.rb

@@ -1,4 +1,5 @@
 copy_file "variants/backend-base/lib/tasks/coverage.rake", "lib/tasks/coverage.rake"
-copy_file "variants/backend-base/lib/tasks/assets.rake", "lib/tasks/assets.rake"
 copy_file "variants/backend-base/lib/tasks/app.rake", "lib/tasks/app.rake"
 copy_file "variants/backend-base/lib/tasks/dev.rake", "lib/tasks/dev.rake"
+
+template "variants/backend-base/lib/tasks/assets.rake.tt", "lib/tasks/assets.rake", force: true