enhancement/issue 684 import meta resolve refactor part 2

CONTRIBUTING.md

@@ -38,7 +38,7 @@ import chai from 'chai';
 import { JSDOM } from 'jsdom';
 import path from 'path';
 import { runSmokeTest } from '../../../../../test/smoke-test.js';
-import { getSetupFiles, getOutputTeardownFiles } from '../../../../../test/utils.js';
+import { getOutputTeardownFiles } from '../../../../../test/utils.js';
 import { Runner } from 'gallinago';
 import { fileURLToPath, URL } from 'url';
 
@@ -60,7 +60,7 @@ describe('Build Greenwood With: ', function() {
   describe(LABEL, function() {
 
     before(function() {
-      runner.setup(outputPath, getSetupFiles(outputPath));
+      runner.setup(outputPath);
       runner.runCommand(cliPath, 'build');
     });
 

packages/cli/package.json

@@ -22,6 +22,7 @@
   "engines": {
     "node": ">=18.20.0"
   },
+  "main": "./src/index.js",
   "bin": {
     "greenwood": "./src/index.js"
   },

packages/cli/src/config/rollup.config.js

@@ -116,30 +116,7 @@
         const resourceKey = normalizePathnameForWindows(resource.sourcePathURL);
 
         for (const bundle in bundles) {
-          let facadeModuleId = (bundles[bundle].facadeModuleId || '').replace(/\\/g, '/');
-          /*
-           * this is an odd issue related to symlinking in our Greenwood monorepo when building the website
-           * and managing packages that we create as "virtual" modules, like for the mpa router
-           *
-           * ex. import @greenwood/router/router.js -> /node_modules/@greenwood/cli/src/lib/router.js
-           *
-           * when running our tests, which better emulates a real user
-           * facadeModuleId will be in node_modules, which is like how it would be for a user:
-           * /node_modules/@greenwood/cli/src/lib/router.js
-           *
-           * however, when building our website, where symlinking points back to our packages/ directory
-           * facadeModuleId will look like this:
-           * /<workspace>/greenwood/packages/cli/src/lib/router.js
-           *
-           * so we need to massage facadeModuleId a bit for Rollup for our internal development
-           * pathToMatch (before): /node_modules/@greenwood/cli/src/lib/router.js
-           * pathToMatch (after): /cli/src/lib/router.js
-           */
-          if (resourceKey?.indexOf('/node_modules/@greenwood/cli') > 0 && facadeModuleId?.indexOf('/packages/cli') > 0) {
-            if (await checkResourceExists(new URL(`file://${facadeModuleId}`))) {
-              facadeModuleId = facadeModuleId.replace('/packages/cli', '/node_modules/@greenwood/cli');
-            }
-          }
+          const facadeModuleId = (bundles[bundle].facadeModuleId || '').replace(/\\/g, '/');
 
           if (resourceKey === facadeModuleId) {
             const { fileName } = bundles[bundle];
@@ -372,7 +349,7 @@
             }
           }
         } else {
           // TODO figure out how to handle URL chunk from SSR pages
           // https://github.com/ProjectEvergreen/greenwood/issues/1163
         }
 

packages/cli/src/lib/node-modules-utils.js

@@ -1,47 +1,23 @@
-import { createRequire } from 'module';
 import { checkResourceExists } from './resource-utils.js';
+import { resolveBareSpecifier, derivePackageRoot } from './walker-package-ranger.js';
 import fs from 'fs/promises';
 
-// TODO delete me and everything else in this file
-// https://github.com/ProjectEvergreen/greenwood/issues/684
-async function getNodeModulesLocationForPackage(packageName) {
-  let nodeModulesUrl;
+// take a "shortcut" pathname, e.g. /node_modules/lit/lit-html.js
+// and resolve it using import.meta.resolve
+function getResolvedHrefFromPathnameShortcut(pathname, rootFallbackUrl) {
+  const segments = pathname.replace('/node_modules/', '').split('/');
+  const hasScope = segments[0].startsWith('@');
+  const specifier = hasScope ? `${segments[0]}/${segments[1]}` : segments[0];
+  const resolved = resolveBareSpecifier(specifier);
 
-  // require.resolve may fail in the event a package has no main in its package.json
-  // so as a fallback, ask for node_modules paths and find its location manually
-  // https://github.com/ProjectEvergreen/greenwood/issues/557#issuecomment-923332104
-  // // https://stackoverflow.com/a/62499498/417806
-  const require = createRequire(import.meta.url);
-  const locations = require.resolve.paths(packageName);
+  if (resolved) {
+    const root = derivePackageRoot(resolved);
 
-  for (const location in locations) {
-    const nodeModulesPackageRoot = `${locations[location]}/${packageName}`;
-    const packageJsonLocation = `${nodeModulesPackageRoot}/package.json`;
-
-    if (await checkResourceExists(new URL(`file://${packageJsonLocation}`))) {
-      nodeModulesUrl = nodeModulesPackageRoot;
-    }
-  }
-
-  if (!nodeModulesUrl) {
-    console.debug(`Unable to look up ${packageName} using NodeJS require.resolve.  Falling back to process.cwd()`);
-    nodeModulesUrl = new URL(`./node_modules/${packageName}`, `file://${process.cwd()}`).pathname;
+    return `${root}${segments.slice(hasScope ? 2 : 1).join('/')}`;
+  } else {
+    // best guess fallback, for example for local theme pack development
+    return new URL(`.${pathname}`, rootFallbackUrl);
   }
-
-  return nodeModulesUrl;
-}
-
-// extract the package name from a URL like /node_modules/<some>/<package>/index.js
-function getPackageNameFromUrl(url) {
-  const packagePathPieces = url.split('node_modules/')[1].split('/'); // double split to handle node_modules within nested paths
-  let packageName = packagePathPieces.shift();
-
-  // handle scoped packages
-  if (packageName.indexOf('@') === 0) {
-    packageName = `${packageName}/${packagePathPieces.shift()}`;
-  }
-
-  return packageName;
 }
 
 async function getPackageJsonForProject({ userWorkspace, projectDirectory }) {
@@ -57,8 +33,37 @@ async function getPackageJsonForProject({ userWorkspace, projectDirectory }) {
       : {};
 }
 
+function mergeImportMap(html = '', map = {}, shouldShim = false) {
+  const importMapType = shouldShim ? 'importmap-shim' : 'importmap';
+  const hasImportMap = html.indexOf(`script type="${importMapType}"`) > 0;
+  const danglingComma = hasImportMap ? ',' : '';
+  const importMap = JSON.stringify(map, null, 2).replace('}', '').replace('{', '');
+
+  if (Object.entries(map).length === 0) {
+    return html;
+  }
+
+  if (hasImportMap) {
+    return html.replace('"imports": {', `
+      "imports": {
+        ${importMap}${danglingComma}
+    `);
+  } else {
+    return html.replace('<head>', `
+      <head>
+      <script type="${importMapType}">
+        {
+          "imports": {
+            ${importMap}
+          }
+        }
+      </script>
+    `);
+  }
+}
+
 export {
   getPackageJsonForProject,
-  getNodeModulesLocationForPackage,
-  getPackageNameFromUrl
+  getResolvedHrefFromPathnameShortcut,
+  mergeImportMap
 };

packages/cli/src/lib/resource-utils.js

@@ -1,15 +1,16 @@
 import fs from 'fs/promises';
 import { hashString } from './hashing-utils.js';
+import { getResolvedHrefFromPathnameShortcut } from '../lib/node-modules-utils.js';
 import htmlparser from 'node-html-parser';
 
 async function modelResource(context, type, src = undefined, contents = undefined, optimizationAttr = undefined, rawAttributes = undefined) {
-  const { projectDirectory, scratchDir, userWorkspace } = context;
+  const { scratchDir, userWorkspace, projectDirectory } = context;
   const extension = type === 'script' ? 'js' : 'css';
   let sourcePathURL;
 
   if (src) {
-    sourcePathURL = src.startsWith('/node_modules')
-      ? new URL(`.${src}`, projectDirectory)
+    sourcePathURL = src.startsWith('/node_modules/')
+      ? new URL(getResolvedHrefFromPathnameShortcut(src, projectDirectory))
       : src.startsWith('/')
         ? new URL(`.${src}`, userWorkspace)
         : new URL(`./${src.replace(/\.\.\//g, '').replace('./', '')}`, userWorkspace);
@@ -40,7 +41,7 @@
   const statusText = source.statusText || destination.statusText;
 
   source.headers.forEach((value, key) => {
     // TODO better way to handle Response automatically setting content-type
     // https://github.com/ProjectEvergreen/greenwood/issues/1049
     const isDefaultHeader = key.toLowerCase() === 'content-type' && value === 'text/plain;charset=UTF-8';
 
@@ -57,7 +58,7 @@
 }
 
 // On Windows, a URL with a drive letter like C:/ thinks it is a protocol and so prepends a /, e.g. /C:/
-// This is fine with never fs methods that Greenwood uses, but tools like Rollup and PostCSS will need this handled manually
+// This is fine with newer fs methods that Greenwood uses, but tools like Rollup and PostCSS will need this handled manually
 // https://github.com/rollup/rollup/issues/3779
 function normalizePathnameForWindows(url) {
   const windowsDriveRegex = /\/[a-zA-Z]{1}:\//;
@@ -170,7 +171,7 @@
   return url !== '' && (url.indexOf('http') !== 0 && url.indexOf('//') !== 0);
 }
 
 // TODO handle full request
 // https://github.com/ProjectEvergreen/greenwood/discussions/1146
 function transformKoaRequestIntoStandardRequest(url, request) {
   const { body, method, header } = request;

packages/cli/src/lib/walker-package-ranger.js

@@ -3,11 +3,14 @@ import fs from 'fs';
 /* eslint-disable max-depth,complexity */
 // priority if from L -> R
 const SUPPORTED_EXPORT_CONDITIONS = ['import', 'module-sync', 'default'];
+const IMPORT_MAP_RESOLVED_PREFIX = '/~';
 const importMap = {};
 const diagnostics = {};
 
-function updateImportMap(key, value) {
-  importMap[key.replace('./', '')] = value.replace('./', '');
+function updateImportMap(key, value, resolvedRoot) {
+  if (!importMap[key.replace('./', '')]) {
+    importMap[key.replace('./', '')] = `${IMPORT_MAP_RESOLVED_PREFIX}${resolvedRoot.replace('file://', '')}${value.replace('./', '')}`;
+  }
 }
 
 // wrapper around import.meta.resolve to provide graceful error handling / logging
@@ -35,11 +38,27 @@ function resolveBareSpecifier(specifier) {
  *   root: 'file:///path/to/project/greenwood-lit-ssr/node_modules/.pnpm/lit-html@3.2.1/node_modules/lit-html/package.json'
  *  }
  */
-function derivePackageRoot(dependencyName, resolved) {
-  const root = resolved.slice(0, resolved.lastIndexOf(`/node_modules/${dependencyName}/`));
-  const derived = `${root}/node_modules/${dependencyName}/`;
+function derivePackageRoot(resolved) {
+  // can't rely on the specifier, for example in monorepos
+  // where @foo/bar may point to a non node_modules location
+  // e.g. packages/some-namespace/package.json
+  // so we walk backwards looking for nearest package.json
+  const segments = resolved
+    .replace('file://', '')
+    .split('/')
+    .filter(segment => segment !== '')
+    .reverse();
+  let root = resolved.replace(segments[0], '');
+
+  for (const segment of segments.slice(1)) {
+    if (fs.existsSync(new URL('./package.json', root))) {
+      break;
+    }
+
+    root = root.replace(`${segment}/`, '');
+  }
 
-  return derived;
+  return root;
 }
 
 // Helper function to convert export patterns to a regex (thanks ChatGPT :D)
@@ -102,33 +121,32 @@ async function walkExportPatterns(dependency, sub, subValue, resolvedRoot) {
       if (stat.isDirectory()) {
         walkDirectoryForExportPatterns(new URL(`./${file}/`, directoryUrl));
       } else if (regexPattern.test(filePathUrl.href)) {
-        const rootSubOffset = patternRoot(sub);
-        const relativePath = filePathUrl.href.replace(resolvedRoot, '/');
+        const relativePath = filePathUrl.href.replace(resolvedRoot, '');
         // naive way to offset a subValue pattern to the sub pattern
         // ex. "./js/*": "./packages/*/src/index.js",
         // https://unpkg.com/browse/@uswds/uswds@3.10.0/package.json
         const rootSubRelativePath = relativePath.replace(rootSubValueOffset, '');
 
-        updateImportMap(`${dependency}${rootSubOffset}${rootSubRelativePath}`, `/node_modules/${dependency}${relativePath}`);
+        updateImportMap(`${dependency}/${rootSubRelativePath}`, relativePath, resolvedRoot);
       }
     });
   }
 
   walkDirectoryForExportPatterns(new URL(`.${rootSubValueOffset}/`, resolvedRoot));
 }
 
-function trackExportConditions(dependency, exports, sub, condition) {
+function trackExportConditions(dependency, exports, sub, condition, resolvedRoot) {
   if (typeof exports[sub] === 'object') {
     // also check for nested conditions of conditions, default to default for now
     // https://unpkg.com/browse/@floating-ui/dom@1.6.12/package.json
     if (sub === '.') {
-      updateImportMap(dependency, `/node_modules/${dependency}/${exports[sub][condition].default ?? exports[sub][condition]}`);
+      updateImportMap(dependency, `${exports[sub][condition].default ?? exports[sub][condition]}`, resolvedRoot);
     } else {
-      updateImportMap(`${dependency}/${sub}`, `/node_modules/${dependency}/${exports[sub][condition].default ?? exports[sub][condition]}`);
+      updateImportMap(`${dependency}/${sub}`, `${exports[sub][condition].default ?? exports[sub][condition]}`, resolvedRoot);
     }
   } else {
     // https://unpkg.com/browse/redux@5.0.1/package.json
-    updateImportMap(dependency, `/node_modules/${dependency}/${exports[sub][condition]}`);
+    updateImportMap(dependency, `${exports[sub][condition]}`);
   }
 }
 
@@ -151,7 +169,7 @@ async function walkPackageForExports(dependency, packageJson, resolvedRoot) {
         for (const condition of SUPPORTED_EXPORT_CONDITIONS) {
           if (exports[sub][condition]) {
             matched = true;
-            trackExportConditions(dependency, exports, sub, condition);
+            trackExportConditions(dependency, exports, sub, condition, resolvedRoot);
             break;
           }
         }
@@ -163,16 +181,21 @@ async function walkPackageForExports(dependency, packageJson, resolvedRoot) {
       } else {
         // handle (unconditional) subpath exports
         if (sub === '.') {
-          updateImportMap(dependency, `/node_modules/${dependency}/${exports[sub]}`);
+          updateImportMap(dependency, `${exports[sub]}`, resolvedRoot);
         } else if (sub.indexOf('*') >= 0) {
           await walkExportPatterns(dependency, sub, exports[sub], resolvedRoot);
         } else {
-          updateImportMap(`${dependency}/${sub}`, `/node_modules/${dependency}/${exports[sub]}`);
+          updateImportMap(`${dependency}/${sub}`, `${exports[sub]}`, resolvedRoot);
         }
       }
     }
   } else if (module || main) {
-    updateImportMap(dependency, `/node_modules/${dependency}/${module ?? main}`);
+    updateImportMap(dependency, `${module ?? main}`, resolvedRoot);
+  } else if (fs.existsSync(new URL('./index.js', resolvedRoot))) {
+    // if an index.js file exists but with no main entry point, then it should count as a main entry point
+    // https://docs.npmjs.com/cli/v7/configuring-npm/package-json#main
+    // https://unpkg.com/browse/object-assign@4.1.1/package.json
+    updateImportMap(dependency, 'index.js', resolvedRoot);
   } else {
     // ex: https://unpkg.com/browse/uuid@3.4.0/package.json
     diagnostics[dependency] = `WARNING: No supported entry point detected for => \`${dependency}\``;
@@ -186,7 +209,7 @@ async function walkPackageJson(packageJson = {}) {
       const resolved = resolveBareSpecifier(dependency);
 
       if (resolved) {
-        const resolvedRoot = derivePackageRoot(dependency, resolved);
+        const resolvedRoot = derivePackageRoot(resolved);
         const resolvedPackageJson = (await import(new URL('./package.json', resolvedRoot), { with: { type: 'json' } })).default;
 
         walkPackageForExports(dependency, resolvedPackageJson, resolvedRoot);
@@ -196,7 +219,7 @@ async function walkPackageJson(packageJson = {}) {
             const resolved = resolveBareSpecifier(dependency);
 
             if (resolved) {
-              const resolvedRoot = derivePackageRoot(dependency, resolved);
+              const resolvedRoot = derivePackageRoot(resolved);
               const resolvedPackageJson = (await import(new URL('./package.json', resolvedRoot), { with: { type: 'json' } })).default;
 
               walkPackageForExports(dependency, resolvedPackageJson, resolvedRoot);
@@ -214,37 +237,9 @@ async function walkPackageJson(packageJson = {}) {
   return { importMap, diagnostics };
 }
 
-// could probably go somewhere else, in a util?
-function mergeImportMap(html = '', map = {}, shouldShim = false) {
-  const importMapType = shouldShim ? 'importmap-shim' : 'importmap';
-  const hasImportMap = html.indexOf(`script type="${importMapType}"`) > 0;
-  const danglingComma = hasImportMap ? ',' : '';
-  const importMap = JSON.stringify(map, null, 2).replace('}', '').replace('{', '');
-
-  if (Object.entries(map).length === 0) {
-    return html;
-  }
-
-  if (hasImportMap) {
-    return html.replace('"imports": {', `
-      "imports": {
-        ${importMap}${danglingComma}
-    `);
-  } else {
-    return html.replace('<head>', `
-      <head>
-      <script type="${importMapType}">
-        {
-          "imports": {
-            ${importMap}
-          }
-        }
-      </script>
-    `);
-  }
-}
-
 export {
   walkPackageJson,
-  mergeImportMap
+  resolveBareSpecifier,
+  derivePackageRoot,
+  IMPORT_MAP_RESOLVED_PREFIX
 };

packages/cli/src/plugins/resource/plugin-active-content.js

@@ -1,4 +1,4 @@
-import { mergeImportMap } from '../../lib/walker-package-ranger.js';
+import { mergeImportMap } from '../../lib/node-modules-utils.js';
 import { ResourceInterface } from '../../lib/resource-interface.js';
 import { checkResourceExists } from '../../lib/resource-utils.js';
 import { activeFrontmatterKeys, cleanContentCollection, pruneGraph } from '../../lib/content-utils.js';