Skip to content

Commit

Permalink
Allow references to files outside js/ and css/ dirs (#26)
Browse files Browse the repository at this point in the history 
Stop marking the directories aside from js/ and css/ as "external" for esbuild. This allows the files inside those other directories to be referenced from within JS/CSS files and properly bundled by esbuild.

Use an `onLoad` esbuild plugin callback to track referenced files so that we can exclude them from separate handling when we copy over the non-referenced asset files.

Specific changes:

- Stop marking any directories as external (in `esbuild.js`)
- Move `findExternalDirectories` from `esbuild.js` to `esbuild-plugin.js` as `extraAssetDirs` (a more appropriate name now that nothing is marked as external), and call this directly when determining which directories we should manually copy via `processAssetDirectory`.
- Introduce an `onLoad` callback to `esbuild-plugin.js` and use this to track any files that esbuild itself loads.
- Use this to skip already-loaded files inside `processAssetDirectory`, which means there will be no double-processing of the files that are referenced from JS/CSS.
- Then when handling esbuild's outputs (i.e. all the loaded files) to include in the manifest, make sure to use the output's original input filename for the manifest key, to avoid collisions and to keep those manifest keys consistent with non-loaded files that are copied across as part of `processAssetDirectory`.
- Plus a range of other refactors/tidyings

Fixes #24.

Co-authored-by: mrimp <krzysiekkamilpiotrek@gmail.com>
  • Loading branch information
@timriley @krzykamil
timriley and krzykamil authored Mar 31, 2024
1 parent 81929ef commit e784a6b
Show file tree
Hide file tree
Showing 5 changed files with 317 additions and 254 deletions.
222 changes: 122 additions & 100 deletions dist/esbuild-plugin.js
View file
Original file line number Diff line number Diff line change
@@ -1,90 +1,112 @@
import fs from "fs-extra";
import path from "path";
import crypto from "node:crypto";
import { globSync } from "glob";
const URL_SEPARATOR = "/";
const assetsDirName = "assets";
const fileHashRegexp = /(-[A-Z0-9]{8})(\.\S+)$/;
const hanamiEsbuild = (options) => {
return {
name: "hanami-esbuild",
setup(build) {
build.initialOptions.metafile = true;
const manifestPath = path.join(options.root, options.destDir, "assets.json");
const externalDirs = build.initialOptions.external || [];
const assetsSourceDir = path.join(options.sourceDir, assetsDirName);
const assetsSourcePath = path.join(options.root, assetsSourceDir);
// Track files loaded by esbuild so we don't double-process them.
const loadedFiles = new Set();
build.onLoad({ filter: /.*/ }, (args) => {
loadedFiles.add(args.path);
return null;
});
// After build, copy over any non-referenced asset files, and create a manifest.
build.onEnd(async (result) => {
const outputs = result.metafile?.outputs;
const assetsManifest = {};
const calulateDestinationUrl = (str) => {
return normalizeUrl(str).replace(/public/, "");
};
const normalizeUrl = (str) => {
return str.replace(/[\\]+/, URL_SEPARATOR);
};
const calculateSubresourceIntegrity = (algorithm, path) => {
const content = fs.readFileSync(path, "utf8");
const hash = crypto.createHash(algorithm).update(content).digest("base64");
return `${algorithm}-${hash}`;
};
// Inspired by https://github.com/evanw/esbuild/blob/2f2b90a99d626921d25fe6d7d0ca50bd48caa427/internal/bundler/bundler.go#L1057
const calculateHash = (hashBytes, hash) => {
if (!hash) {
return null;
const manifest = {};
if (typeof outputs === "undefined") {
return;
}
// Copy extra asset files (in dirs besides js/ and css/) into the destination directory
const copiedAssets = [];
assetDirectories().forEach((dir) => {
copiedAssets.push(...processAssetDirectory(dir));
});
// Add copied assets into the manifest
for (const copiedAsset of copiedAssets) {
if (copiedAsset.sourcePath.endsWith(".map")) {
continue;
}
const result = crypto.createHash("sha256").update(hashBytes).digest("hex");
return result.slice(0, 8).toUpperCase();
};
// Transforms the esbuild metafile outputs into an object containing mappings of outputs
// generated from entryPoints only.
//
// Converts this:
//
// {
// 'public/assets/admin/app-ITGLRDE7.js': {
// imports: [],
// exports: [],
// entryPoint: 'slices/admin/assets/js/app.js',
// inputs: { 'slices/admin/assets/js/app.js': [Object] },
// bytes: 95
// }
// }
//
// To this:
//
// {
// 'public/assets/admin/app-ITGLRDE7.js': true
// }
function extractEsbuildCompiledEntrypoints(esbuildOutputs) {
const entryPoints = {};
for (const key in esbuildOutputs) {
if (!key.endsWith(".map")) {
entryPoints[key] = true;
}
// Take the full path of the copied asset and remove everything up to (and including) the "assets/" dir
var sourceUrl = copiedAsset.sourcePath.replace(assetsSourcePath + path.sep, "");
// Then remove the first subdir (e.g. "images/"), since we do not include those in the asset paths
sourceUrl = sourceUrl.substring(sourceUrl.indexOf("/") + 1);
manifest[sourceUrl] = prepareAsset(copiedAsset.destPath);
}
// Add files already bundled by esbuild into the manifest
for (const outputFile in outputs) {
if (outputFile.endsWith(".map")) {
continue;
}
const outputAttrs = outputs[outputFile];
const inputFiles = Object.keys(outputAttrs.inputs);
// Determine the manifest key for the esbuild output file
let manifestKey;
if (!(outputFile.endsWith(".js") || outputFile.endsWith(".css")) &&
inputFiles.length == 1 &&
inputFiles[0].startsWith(assetsSourceDir + path.sep)) {
// A non-JS/CSS output with a single input will be an asset file that has been been
// referenced from JS/CSS.
//
// In this case, preserve the original input file's path in the manifest key, so it
// matches any other files copied over from that path via processAssetDirectory.
//
// For example, given the input file "app/assets/images/icons/some-icon.png", return a
// manifest key of "icons/some-icon.png".
manifestKey = inputFiles[0]
.substring(assetsSourceDir.length + 1) // + 1 to account for the sep
.split(path.sep)
.slice(1)
.join(path.sep);
}
return entryPoints;
else {
// For all other outputs, determine the manifest key based on the output file name,
// stripping away the hash suffix added by esbuild.
//
// For example, given the output "public/assets/app-2TLUHCQ6.js", return an manifest
// key of "app.js".
manifestKey = outputFile
.replace(options.destDir + path.sep, "")
.replace(fileHashRegexp, "$2");
}
manifest[manifestKey] = prepareAsset(outputFile);
}
// TODO: profile the current implementation vs blindly copying the asset
const copyAsset = (srcPath, destPath) => {
if (fs.existsSync(destPath)) {
const srcStat = fs.statSync(srcPath);
const destStat = fs.statSync(destPath);
// File already exists and is up-to-date, skip copying
if (srcStat.mtimeMs <= destStat.mtimeMs) {
return;
}
// Write assets manifest to the destination directory
await fs.writeJson(manifestPath, manifest, { spaces: 2 });
//
// Helper functions
//
function assetDirectories() {
const excludeDirs = ["js", "css"];
try {
const dirs = globSync([path.join(assetsSourcePath, "*")], { nodir: false });
const filteredDirs = dirs.filter((dir) => {
const dirName = dir.split(path.sep).pop();
return !excludeDirs.includes(dirName);
});
return filteredDirs;
}
if (!fs.existsSync(path.dirname(destPath))) {
fs.mkdirSync(path.dirname(destPath), { recursive: true });
catch (err) {
console.error("Error listing external directories:", err);
return [];
}
fs.copyFileSync(srcPath, destPath);
return;
};
const processAssetDirectory = (pattern, compiledEntryPoints, options) => {
const dirPath = path.dirname(pattern);
const files = fs.readdirSync(dirPath, { recursive: true });
}
function processAssetDirectory(assetDir) {
const files = fs.readdirSync(assetDir, { recursive: true });
const assets = [];
files.forEach((file) => {
const sourcePath = path.join(dirPath, file.toString());
// Skip if the file is already processed by esbuild
if (compiledEntryPoints.hasOwnProperty(sourcePath)) {
const sourcePath = path.join(assetDir, file.toString());
// Skip files loaded by esbuild; those are added to the manifest separately
if (loadedFiles.has(sourcePath)) {
return;
}
// Skip directories and any other non-files
Expand All @@ -96,28 +118,36 @@ const hanamiEsbuild = (options) => {
const baseName = path.basename(sourcePath, fileExtension);
const destFileName = [baseName, fileHash].filter((item) => item !== null).join("-") + fileExtension;
const destPath = path.join(options.destDir, path
.relative(dirPath, sourcePath)
.relative(assetDir, sourcePath)
.replace(path.basename(file.toString()), destFileName));
if (fs.lstatSync(sourcePath).isDirectory()) {
assets.push(...processAssetDirectory(destPath, compiledEntryPoints, options));
assets.push(...processAssetDirectory(destPath));
}
else {
copyAsset(sourcePath, destPath);
assets.push({ sourcePath: sourcePath, destPath: destPath });
}
});
return assets;
};
if (typeof outputs === "undefined") {
}
// TODO: profile the current implementation vs blindly copying the asset
function copyAsset(srcPath, destPath) {
if (fs.existsSync(destPath)) {
const srcStat = fs.statSync(srcPath);
const destStat = fs.statSync(destPath);
// File already exists and is up-to-date, skip copying
if (srcStat.mtimeMs <= destStat.mtimeMs) {
return;
}
}
if (!fs.existsSync(path.dirname(destPath))) {
fs.mkdirSync(path.dirname(destPath), { recursive: true });
}
fs.copyFileSync(srcPath, destPath);
return;
}
const compiledEntryPoints = extractEsbuildCompiledEntrypoints(outputs);
const copiedAssets = [];
externalDirs.forEach((pattern) => {
copiedAssets.push(...processAssetDirectory(pattern, compiledEntryPoints, options));
});
function prepareAsset(assetPath, destinationUrl) {
var asset = { url: destinationUrl };
function prepareAsset(assetPath) {
var asset = { url: calculateDestinationUrl(assetPath) };
if (options.sriAlgorithms.length > 0) {
asset.sri = [];
for (const algorithm of options.sriAlgorithms) {
Expand All @@ -127,31 +157,23 @@ const hanamiEsbuild = (options) => {
}
return asset;
}
// Process entrypoints
const fileHashRegexp = /(-[A-Z0-9]{8})(\.\S+)$/;
for (const compiledEntryPoint in compiledEntryPoints) {
// Convert "public/assets/app-2TLUHCQ6.js" to "app.js"
let sourceUrl = compiledEntryPoint
.replace(options.destDir + "/", "")
.replace(fileHashRegexp, "$2");
const destinationUrl = calulateDestinationUrl(compiledEntryPoint);
assetsManifest[sourceUrl] = prepareAsset(compiledEntryPoint, destinationUrl);
function calculateDestinationUrl(str) {
const normalizedUrl = str.replace(/[\\]+/, URL_SEPARATOR);
return normalizedUrl.replace(/public/, "");
}
// Process copied assets
for (const copiedAsset of copiedAssets) {
// TODO: I wonder if we can skip .map files earlier
if (copiedAsset.sourcePath.endsWith(".map")) {
continue;
function calculateSubresourceIntegrity(algorithm, path) {
const content = fs.readFileSync(path, "utf8");
const hash = crypto.createHash(algorithm).update(content).digest("base64");
return `${algorithm}-${hash}`;
}
// Inspired by https://github.com/evanw/esbuild/blob/2f2b90a99d626921d25fe6d7d0ca50bd48caa427/internal/bundler/bundler.go#L1057
function calculateHash(hashBytes, hash) {
if (!hash) {
return null;
}
const destinationUrl = calulateDestinationUrl(copiedAsset.destPath);
// Take the full path of the copied asset and remove everything up to (and including) the "assets/" dir
var sourceUrl = copiedAsset.sourcePath.replace(path.join(options.root, options.sourceDir, assetsDirName) + "/", "");
// Then remove the first subdir (e.g. "images/"), since we do not include those in the asset paths
sourceUrl = sourceUrl.substring(sourceUrl.indexOf("/") + 1);
assetsManifest[sourceUrl] = prepareAsset(copiedAsset.destPath, destinationUrl);
const result = crypto.createHash("sha256").update(hashBytes).digest("hex");
return result.slice(0, 8).toUpperCase();
}
// Write assets manifest to the destination directory
await fs.writeJson(manifestPath, assetsManifest, { spaces: 2 });
});
},
};
Expand Down
17 changes: 0 additions & 17 deletions dist/esbuild.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,22 +38,6 @@ const findEntryPoints = (sliceRoot) => {
});
return result;
};
const findExternalDirectories = (basePath) => {
const assetDirsPattern = [path.join(basePath, assetsDirName, "*")];
const excludeDirs = ["js", "css"];
try {
const dirs = globSync(assetDirsPattern, { nodir: false });
const filteredDirs = dirs.filter((dir) => {
const dirName = dir.split(path.sep).pop();
return !excludeDirs.includes(dirName);
});
return filteredDirs.map((dir) => path.join(dir, "*"));
}
catch (err) {
console.error("Error listing external directories:", err);
return [];
}
};
const commonPluginOptions = (root, args) => {
return {
root: root,
Expand All @@ -69,7 +53,6 @@ const commonOptions = (root, args, plugin) => {
outdir: args.dest,
absWorkingDir: root,
loader: loader,
external: findExternalDirectories(path.join(root, args.path)),
logLevel: "info",
entryPoints: findEntryPoints(path.join(root, args.path)),
plugins: [plugin],
Expand Down
Loading

0 comments on commit e784a6b

Please sign in to comment.