From 99455b48f3a3be54baff4ded21a80a337e63caf9 Mon Sep 17 00:00:00 2001 From: Olen Davis Date: Wed, 15 Jan 2025 13:01:36 -0800 Subject: [PATCH 1/4] fix: caching refactor (WIP) --- src/index.js | 11 ++++++-- src/loader.js | 75 ++++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 68 insertions(+), 18 deletions(-) diff --git a/src/index.js b/src/index.js index f5b75bc..f4e325c 100644 --- a/src/index.js +++ b/src/index.js @@ -246,6 +246,7 @@ class ImageMinimizerPlugin { } const cache = compilation.getCache("ImageMinimizerWebpackPlugin"); + const assetsForTransformers = ( await Promise.all( Object.keys(assets) @@ -327,7 +328,7 @@ class ImageMinimizerPlugin { 1, this.options.concurrency ?? os.cpus()?.length ?? 1, ); - const { RawSource } = compiler.webpack.sources; + const { RawSource, CachedSource } = compiler.webpack.sources; const scheduledTasks = assetsForTransformers.map((asset) => async () => { const { name, info, inputSource, cacheItem, transformer } = asset; @@ -336,7 +337,11 @@ class ImageMinimizerPlugin { const sourceFromInputSource = inputSource.source(); + const pluginName = this.constructor.name; + const logger = compilation.getLogger(pluginName); + if (!output) { + logger.log(`cache miss: ${name}`); input = sourceFromInputSource; if (!Buffer.isBuffer(input)) { @@ -356,7 +361,7 @@ class ImageMinimizerPlugin { output = await worker(minifyOptions); - output.source = new RawSource(output.data); + output.source = new CachedSource(new RawSource(output.data)); await cacheItem.storePromise({ source: output.source, @@ -365,6 +370,8 @@ class ImageMinimizerPlugin { warnings: output.warnings, errors: output.errors, }); + } else { + logger.log(`cache miss: ${name}`); } compilation.warnings = [ diff --git a/src/loader.js b/src/loader.js index 945fc7f..aea7ff9 100644 --- a/src/loader.js +++ b/src/loader.js @@ -2,7 +2,9 @@ const path = require("path"); const worker = require("./worker"); const schema = require("./loader-options.json"); -const { isAbsoluteURL } = require("./utils.js"); +const { isAbsoluteURL, memoize } = require("./utils.js"); + +const getSerializeJavascript = memoize(() => require("serialize-javascript")); /** @typedef {import("schema-utils/declarations/validate").Schema} Schema */ /** @typedef {import("webpack").Compilation} Compilation */ @@ -202,18 +204,48 @@ async function loader(content) { ? this.resourcePath : path.relative(this.rootContext, this.resourcePath); - const minifyOptions = - /** @type {import("./index").InternalWorkerOptions} */ ({ - input: content, - filename, - severityError, - transformer, - generateFilename: - /** @type {Compilation} */ - (this._compilation).getAssetPath.bind(this._compilation), - }); + if (!this._compilation || !this._compiler) { + callback(new Error("_compilation and/or _compiler unavailable")); + return; + } + + const logger = this._compilation.getLogger("ImageMinimizerPlugin"); + const cache = this._compilation.getCache("ImageMinimizerWebpackPlugin"); - const output = await worker(minifyOptions); + const cacheName = getSerializeJavascript()({ + name: this.resourcePath, + transformer, + }); + const { RawSource, CachedSource } = this._compiler.webpack.sources; + const eTag = cache.getLazyHashedEtag( + new CachedSource(new RawSource(content)), + ); + const cacheItem = cache.getItemCache(cacheName, eTag); + const output = await cacheItem.providePromise(async () => { + const minifyOptions = + /** @type {import("./index").InternalWorkerOptions} */ ({ + input: content, + filename, + severityError, + transformer, + generateFilename: + /** @type {Compilation} */ + (this._compilation).getAssetPath.bind(this._compilation), + }); + + logger.warn(`Cache miss: ${filename}`); + this.emitWarning(new Error(`Cache miss: ${filename}`)); + + const { data, info, warnings, errors } = await worker(minifyOptions); + + return { + source: new CachedSource(new RawSource(data)), + info, + filename, + warnings, + errors, + }; + }); if (output.errors && output.errors.length > 0) { for (const error of output.errors) { @@ -238,8 +270,8 @@ async function loader(content) { ); this._module.resourceResolveData.encodedContent = isBase64 - ? output.data.toString("base64") - : encodeURIComponent(output.data.toString("utf-8")).replace( + ? output.source.buffer().toString("base64") + : encodeURIComponent(output.source.buffer().toString("utf-8")).replace( /[!'()*]/g, (character) => `%${/** @type {number} */ (character.codePointAt(0)).toString(16)}`, @@ -259,7 +291,18 @@ async function loader(content) { isAbsolute = isAbsoluteURL(output.filename); // Old approach for `file-loader` and other old loaders - changeResource(this, isAbsolute, output, query); + changeResource( + this, + isAbsolute, + { + data: output.source.buffer(), + info: output.info, + filename: output.filename, + errors: output.errors, + warnings: output.warnings, + }, + query, + ); // Change name of assets modules after generator if (this._module && !this._module.matchResource) { @@ -275,7 +318,7 @@ async function loader(content) { }; } - callback(null, output.data); + callback(null, output.source.buffer()); } loader.raw = true; From ccf3d8eebb92650ab104ad7cc9b476e10fecb3d7 Mon Sep 17 00:00:00 2001 From: Olen Davis Date: Thu, 16 Jan 2025 17:32:50 -0800 Subject: [PATCH 2/4] refactor: fixes and tests the providePromise cache implementation to work on my specific project Breaks multiple (not all) tests; working on fixing those next. --- package.json | 5 ++++- src/index.js | 36 +++++++++++++++++++++++------------- src/loader.js | 19 ++++++++++++------- 3 files changed, 39 insertions(+), 21 deletions(-) diff --git a/package.json b/package.json index 639a5a2..fc502c7 100644 --- a/package.json +++ b/package.json @@ -17,12 +17,15 @@ "node": ">= 18.12.0" }, "scripts": { - "start": "npm run build -- -w", + "start": "npm-run-all -p \"watch:**\"", "clean": "del-cli dist types", "prebuild": "npm run clean", "build:types": "tsc --declaration --emitDeclarationOnly && prettier \"types/**/*.ts\" --write", + "postbuild:types": "prettier \"types/**/*.ts\" --write", "build:code": "cross-env NODE_ENV=production babel src -d dist --copy-files", "build": "npm-run-all -p \"build:**\"", + "watch:types": "npm run build:types -- -w", + "watch:code": "npm run build:code -- -w", "commitlint": "commitlint --from=master", "security": "npm audit --production", "lint:prettier": "prettier --cache --list-different .", diff --git a/src/index.js b/src/index.js index 6ce93b5..76899fd 100644 --- a/src/index.js +++ b/src/index.js @@ -282,9 +282,19 @@ class ImageMinimizerPlugin { * @returns {Promise>} */ const getFromCache = async (transformer) => { - const cacheName = getSerializeJavascript()({ name, transformer }); const eTag = cache.getLazyHashedEtag(source); - const cacheItem = cache.getItemCache(cacheName, eTag); + const cacheName = getSerializeJavascript()({ + eTag: eTag.toString(), + transformer: (Array.isArray(transformer) + ? transformer + : [transformer] + ).map(({ implementation, options }) => ({ + implementation, + options, + })), + }); + + const cacheItem = cache.getItemCache(cacheName, null); const output = await cacheItem.getPromise(); return { @@ -343,7 +353,6 @@ class ImageMinimizerPlugin { const logger = compilation.getLogger(pluginName); if (!output) { - logger.log(`cache miss: ${name}`); input = sourceFromInputSource; if (!Buffer.isBuffer(input)) { @@ -361,19 +370,20 @@ class ImageMinimizerPlugin { generateFilename: compilation.getAssetPath.bind(compilation), }); - output = await worker(minifyOptions); + output = await cacheItem.providePromise(async () => { + logger.debug(`optimize cache miss: ${name}`); - output.source = new CachedSource(new RawSource(output.data)); + const result = await worker(minifyOptions); - await cacheItem.storePromise({ - source: output.source, - info: output.info, - filename: output.filename, - warnings: output.warnings, - errors: output.errors, + return { + data: result.data, + source: new CachedSource(new RawSource(result.data)), + info: result.info, + filename: result.filename, + warnings: result.warnings, + errors: result.errors, + }; }); - } else { - logger.log(`cache miss: ${name}`); } compilation.warnings = [ diff --git a/src/loader.js b/src/loader.js index c2410ca..f1b5113 100644 --- a/src/loader.js +++ b/src/loader.js @@ -220,15 +220,20 @@ async function loader(content) { const logger = this._compilation.getLogger("ImageMinimizerPlugin"); const cache = this._compilation.getCache("ImageMinimizerWebpackPlugin"); - const cacheName = getSerializeJavascript()({ - name: this.resourcePath, - transformer, - }); const { RawSource, CachedSource } = this._compiler.webpack.sources; const eTag = cache.getLazyHashedEtag( new CachedSource(new RawSource(content)), ); - const cacheItem = cache.getItemCache(cacheName, eTag); + const cacheName = getSerializeJavascript()({ + eTag: eTag.toString(), + transformer: (Array.isArray(transformer) ? transformer : [transformer]).map( + (each) => ({ + implementation: each.implementation, + options: each.options, + }), + ), + }); + const cacheItem = cache.getItemCache(cacheName, null); const output = await cacheItem.providePromise(async () => { const minifyOptions = /** @type {import("./index").InternalWorkerOptions} */ ({ @@ -241,12 +246,12 @@ async function loader(content) { (this._compilation).getAssetPath.bind(this._compilation), }); - logger.warn(`Cache miss: ${filename}`); - this.emitWarning(new Error(`Cache miss: ${filename}`)); + logger.debug(`loader cache miss: ${filename}`); const { data, info, warnings, errors } = await worker(minifyOptions); return { + data, source: new CachedSource(new RawSource(data)), info, filename, From 542ad09fc274ae993532a50712dfa2e187905b17 Mon Sep 17 00:00:00 2001 From: Olen Davis Date: Fri, 17 Jan 2025 12:36:01 -0800 Subject: [PATCH 3/4] test: fixes tests for refactors --- src/index.js | 7 +++---- src/loader.js | 28 ++++++++-------------------- 2 files changed, 11 insertions(+), 24 deletions(-) diff --git a/src/index.js b/src/index.js index 76899fd..27474f6 100644 --- a/src/index.js +++ b/src/index.js @@ -248,7 +248,6 @@ class ImageMinimizerPlugin { } const cache = compilation.getCache("ImageMinimizerWebpackPlugin"); - const assetsForTransformers = ( await Promise.all( Object.keys(assets) @@ -294,7 +293,7 @@ class ImageMinimizerPlugin { })), }); - const cacheItem = cache.getItemCache(cacheName, null); + const cacheItem = cache.getItemCache(cacheName, eTag); const output = await cacheItem.getPromise(); return { @@ -340,7 +339,7 @@ class ImageMinimizerPlugin { 1, this.options.concurrency ?? os.cpus()?.length ?? 1, ); - const { RawSource, CachedSource } = compiler.webpack.sources; + const { RawSource } = compiler.webpack.sources; const scheduledTasks = assetsForTransformers.map((asset) => async () => { const { name, info, inputSource, cacheItem, transformer } = asset; @@ -377,7 +376,7 @@ class ImageMinimizerPlugin { return { data: result.data, - source: new CachedSource(new RawSource(result.data)), + source: new RawSource(result.data), info: result.info, filename: result.filename, warnings: result.warnings, diff --git a/src/loader.js b/src/loader.js index f1b5113..c734ea4 100644 --- a/src/loader.js +++ b/src/loader.js @@ -220,10 +220,8 @@ async function loader(content) { const logger = this._compilation.getLogger("ImageMinimizerPlugin"); const cache = this._compilation.getCache("ImageMinimizerWebpackPlugin"); - const { RawSource, CachedSource } = this._compiler.webpack.sources; - const eTag = cache.getLazyHashedEtag( - new CachedSource(new RawSource(content)), - ); + const { RawSource } = this._compiler.webpack.sources; + const eTag = cache.getLazyHashedEtag(new RawSource(content)); const cacheName = getSerializeJavascript()({ eTag: eTag.toString(), transformer: (Array.isArray(transformer) ? transformer : [transformer]).map( @@ -233,8 +231,8 @@ async function loader(content) { }), ), }); - const cacheItem = cache.getItemCache(cacheName, null); - const output = await cacheItem.providePromise(async () => { + const cacheItem = cache.getItemCache(cacheName, eTag); + const output = await cacheItem.providePromise(() => { const minifyOptions = /** @type {import("./index").InternalWorkerOptions} */ ({ input: content, @@ -248,16 +246,7 @@ async function loader(content) { logger.debug(`loader cache miss: ${filename}`); - const { data, info, warnings, errors } = await worker(minifyOptions); - - return { - data, - source: new CachedSource(new RawSource(data)), - info, - filename, - warnings, - errors, - }; + return worker(minifyOptions); }); if (output.errors && output.errors.length > 0) { @@ -283,8 +272,8 @@ async function loader(content) { ); this._module.resourceResolveData.encodedContent = isBase64 - ? output.source.buffer().toString("base64") - : encodeURIComponent(output.source.buffer().toString("utf-8")).replace( + ? output.data.toString("base64") + : encodeURIComponent(output.data.toString("utf-8")).replace( /[!'()*]/g, (character) => `%${/** @type {number} */ (character.codePointAt(0)).toString(16)}`, @@ -303,7 +292,6 @@ async function loader(content) { } // Old approach for `file-loader` and other old loaders - // @ts-ignore changeResource(this, output, query); // Change name of assets modules after generator @@ -316,7 +304,7 @@ async function loader(content) { IMAGE_MINIMIZER_PLUGIN_INFO_MAPPINGS.set(this._module, output.info); } - callback(null, output.source.buffer()); + callback(null, output.data); } loader.raw = true; From b738fa46e258bde141f0876860e386f64dcb3d18 Mon Sep 17 00:00:00 2001 From: Olen Davis Date: Mon, 20 Jan 2025 15:09:28 -0800 Subject: [PATCH 4/4] chore: cleanup mistake from merge --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index fc502c7..d4c6f97 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "start": "npm-run-all -p \"watch:**\"", "clean": "del-cli dist types", "prebuild": "npm run clean", - "build:types": "tsc --declaration --emitDeclarationOnly && prettier \"types/**/*.ts\" --write", + "build:types": "tsc --declaration --emitDeclarationOnly", "postbuild:types": "prettier \"types/**/*.ts\" --write", "build:code": "cross-env NODE_ENV=production babel src -d dist --copy-files", "build": "npm-run-all -p \"build:**\"",