From 90fa704787fc177ddfa2bd80ed2350509433b548 Mon Sep 17 00:00:00 2001 From: prisis Date: Thu, 19 Sep 2024 22:36:03 +0200 Subject: [PATCH] fix: improved the file-cache --- .../__tests__/unit/utils/file-cache.test.ts | 50 ++++++--------- packages/packem/src/create-bundler.ts | 47 ++++++++++---- packages/packem/src/utils/file-cache.ts | 64 ++++++++++++++----- 3 files changed, 104 insertions(+), 57 deletions(-) diff --git a/packages/packem/__tests__/unit/utils/file-cache.test.ts b/packages/packem/__tests__/unit/utils/file-cache.test.ts index 6957de6ff..0d1c7bd78 100644 --- a/packages/packem/__tests__/unit/utils/file-cache.test.ts +++ b/packages/packem/__tests__/unit/utils/file-cache.test.ts @@ -1,8 +1,8 @@ import { rm } from "node:fs/promises"; -import { findCacheDirSync } from "@visulima/find-cache-dir"; import { isAccessibleSync, readFileSync } from "@visulima/fs"; import type { Pail } from "@visulima/pail"; +import { join } from "@visulima/path"; import { temporaryDirectory } from "tempy"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; @@ -10,27 +10,22 @@ import FileCache from "../../../src/utils/file-cache"; const hoisted = vi.hoisted(() => { return { - cachePath: vi.fn(), fs: { isAccessibleSync: vi.fn(), readFileSync: vi.fn(), writeFileSync: vi.fn() }, logger: { debug: vi.fn(), warn: vi.fn() } as unknown as Pail, }; }); -vi.mock("@visulima/find-cache-dir", () => { - return { findCacheDirSync: vi.fn().mockReturnValue(hoisted.cachePath) }; -}); - vi.mock("@visulima/fs", () => { return { isAccessibleSync: hoisted.fs.isAccessibleSync, readFileSync: hoisted.fs.readFileSync, writeFileSync: hoisted.fs.writeFileSync }; }); describe("fileCache", () => { let temporaryDirectoryPath: string; + let cacheDirectoryPath: string; beforeEach(async () => { temporaryDirectoryPath = temporaryDirectory(); - - vi.mocked(findCacheDirSync).mockReturnValue(temporaryDirectoryPath); + cacheDirectoryPath = join(temporaryDirectoryPath, "cache") }); afterEach(async () => { @@ -41,35 +36,30 @@ describe("fileCache", () => { expect.assertions(1); // eslint-disable-next-line no-new - new FileCache(temporaryDirectoryPath, "hash123", hoisted.logger); + new FileCache(temporaryDirectoryPath, cacheDirectoryPath, "hash123", hoisted.logger); - expect(hoisted.logger.debug).toHaveBeenCalledWith(expect.stringContaining("Cache path is: " + temporaryDirectoryPath)); + expect(hoisted.logger.debug).toHaveBeenCalledWith({ + message: "Cache path is: " + cacheDirectoryPath, + prefix: "file-cache" + }); }); it("should update isEnabled state when setter is called", () => { expect.assertions(1); - const fileCache = new FileCache(temporaryDirectoryPath, "hash123", hoisted.logger); + const fileCache = new FileCache(temporaryDirectoryPath, cacheDirectoryPath, "hash123", hoisted.logger); fileCache.isEnabled = false; expect(fileCache.isEnabled).toBeFalsy(); }); - it("should return correct cache path when getter is called", () => { - expect.assertions(1); - - const fileCache = new FileCache(temporaryDirectoryPath, "hash123", hoisted.logger); - - expect(fileCache.cachePath).toBeDefined(); - }); - it("should return true if file is accessible in the cache", () => { expect.assertions(1); vi.mocked(isAccessibleSync).mockReturnValue(true); - const fileCache = new FileCache(temporaryDirectoryPath, "hash123", hoisted.logger); + const fileCache = new FileCache(temporaryDirectoryPath, cacheDirectoryPath, "hash123", hoisted.logger); expect(fileCache.has("testFile")).toBeTruthy(); }); @@ -77,7 +67,7 @@ describe("fileCache", () => { it("should retrieve data from memory cache if available", () => { expect.assertions(3); - const fileCache = new FileCache(temporaryDirectoryPath, "hash123", hoisted.logger); + const fileCache = new FileCache(temporaryDirectoryPath, cacheDirectoryPath, "hash123", hoisted.logger); const data = { key: "value" }; vi.mocked(isAccessibleSync).mockReturnValue(true); @@ -96,7 +86,7 @@ describe("fileCache", () => { vi.mocked(readFileSync).mockReturnValue(jsonData); vi.mocked(isAccessibleSync).mockReturnValue(true); - const fileCache = new FileCache(temporaryDirectoryPath, "hash123", hoisted.logger); + const fileCache = new FileCache(temporaryDirectoryPath, cacheDirectoryPath, "hash123", hoisted.logger); expect(fileCache.get("/path/to/file")).toStrictEqual({ key: "value" }); }); @@ -104,11 +94,13 @@ describe("fileCache", () => { it("should handle undefined cache path gracefully in constructor", () => { expect.assertions(1); - vi.mocked(findCacheDirSync).mockReturnValue(undefined); // eslint-disable-next-line no-new - new FileCache(temporaryDirectoryPath, "hash123", hoisted.logger); + new FileCache(temporaryDirectoryPath, undefined, "hash123", hoisted.logger); - expect(hoisted.logger.debug).toHaveBeenCalledWith("Could not create cache directory."); + expect(hoisted.logger.debug).toHaveBeenCalledWith({ + message: "Could not create cache directory.", + prefix: "file-cache" + }); }); it("should handle non-JSON data correctly in get method", () => { @@ -119,7 +111,7 @@ describe("fileCache", () => { vi.mocked(readFileSync).mockReturnValue(nonJsonData); vi.mocked(isAccessibleSync).mockReturnValue(true); - const fileCache = new FileCache(temporaryDirectoryPath, "hash123", hoisted.logger); + const fileCache = new FileCache(temporaryDirectoryPath, cacheDirectoryPath, "hash123", hoisted.logger); expect(fileCache.get("/path/to/file")).toStrictEqual(nonJsonData); }); @@ -127,14 +119,14 @@ describe("fileCache", () => { it("should handle undefined data input gracefully in set method", () => { expect.assertions(1); - const fileCache = new FileCache(temporaryDirectoryPath, "hash123", hoisted.logger); + const fileCache = new FileCache(temporaryDirectoryPath, cacheDirectoryPath, "hash123", hoisted.logger); expect(() => fileCache.set("testFile", undefined)).not.toThrow(); }); it("should return false if cache is disabled in has method", () => { expect.assertions(1); - const fileCache = new FileCache(temporaryDirectoryPath, "hash123", hoisted.logger); + const fileCache = new FileCache(temporaryDirectoryPath, cacheDirectoryPath, "hash123", hoisted.logger); fileCache.isEnabled = false; expect(fileCache.has("testFile")).toBeFalsy(); }); @@ -142,7 +134,7 @@ describe("fileCache", () => { it("should return undefined if cache is disabled in get method", () => { expect.assertions(1); - const fileCache = new FileCache(temporaryDirectoryPath, "hash123", hoisted.logger); + const fileCache = new FileCache(temporaryDirectoryPath, cacheDirectoryPath, "hash123", hoisted.logger); fileCache.isEnabled = false; expect(fileCache.get("testFile")).toBeUndefined(); }); diff --git a/packages/packem/src/create-bundler.ts b/packages/packem/src/create-bundler.ts index d1d374af9..1f92a4808 100644 --- a/packages/packem/src/create-bundler.ts +++ b/packages/packem/src/create-bundler.ts @@ -1,8 +1,11 @@ +import { readdirSync } from "node:fs"; +import { rm } from "node:fs/promises"; import Module from "node:module"; import { cwd } from "node:process"; import { bold, cyan } from "@visulima/colorize"; -import { emptyDir, ensureDirSync, isAccessible, isAccessibleSync } from "@visulima/fs"; +import { findCacheDirSync } from "@visulima/find-cache-dir"; +import { emptyDir, ensureDirSync, isAccessible, isAccessibleSync, readJsonSync } from "@visulima/fs"; import { duration } from "@visulima/humanizer"; import type { PackageJson } from "@visulima/package"; import type { Pail } from "@visulima/pail"; @@ -500,6 +503,31 @@ const cleanDistributionDirectories = async (context: BuildContext): Promise => { + if (cachePath && isAccessibleSync(join(cachePath, "keystore.json"))) { + const keyStore: Record = readJsonSync(join(cachePath, "keystore.json")); + + // eslint-disable-next-line security/detect-non-literal-fs-filename + const cacheDirectories = readdirSync(cachePath, { + withFileTypes: true, + }).filter((dirent) => dirent.isDirectory()); + + for await (const dirent of cacheDirectories) { + if (!keyStore[dirent.name]) { + await rm(join(cachePath, dirent.name), { + force: true, + recursive: true, + }); + + logger.info({ + message: "Removing " + dirent.name + " file cache, the cache key is not used anymore.", + prefix: "file-cache" + }) + } + } + } +}; + const createBundler = async ( rootDirectory: string, mode: Mode, @@ -608,18 +636,13 @@ const createBundler = async ( }), ); + const cachePath = findCacheDirSync(packageJson.name as string, { + cwd: rootDirectory, + }); + for await (const config of arrayify(buildConfig)) { const cacheKey = packageJsonCacheKey + getHash(JSON.stringify(config)); - - const fileCache = new FileCache(rootDirectory, cacheKey, logger); - - // clear cache if the cache key has changed - if (fileCache.cachePath && !isAccessibleSync(join(fileCache.cachePath, cacheKey)) && isAccessibleSync(fileCache.cachePath)) { - logger.info("Clearing file cache because the cache key has changed."); - - await emptyDir(fileCache.cachePath); - } - + const fileCache = new FileCache(rootDirectory, cachePath, cacheKey, logger); const context = await createContext(logger, rootDirectory, mode, environment, debug ?? false, restInputConfig, config, packageJson, tsconfig, jiti); fileCache.isEnabled = context.options.fileCache as boolean; @@ -670,6 +693,8 @@ const createBundler = async ( logger.raw("\n⚡️ Build run in " + getDuration()); } + await removeOldCacheFolders(cachePath, logger); + // Restore all wrapped console methods logger.restoreAll(); // eslint-disable-next-line @typescript-eslint/no-explicit-any diff --git a/packages/packem/src/utils/file-cache.ts b/packages/packem/src/utils/file-cache.ts index 9e8f3c4ca..691a105b6 100644 --- a/packages/packem/src/utils/file-cache.ts +++ b/packages/packem/src/utils/file-cache.ts @@ -1,5 +1,4 @@ -import { findCacheDirSync } from "@visulima/find-cache-dir"; -import { isAccessibleSync, readFileSync, writeFileSync } from "@visulima/fs"; +import { isAccessibleSync, readFileSync, readJsonSync, writeFileSync, writeJsonSync } from "@visulima/fs"; import type { Pail } from "@visulima/pail"; import { join, toNamespacedPath } from "@visulima/path"; @@ -18,35 +17,37 @@ class FileCache { readonly #cachePath: undefined | string; - readonly #packageJsonHash: string; + readonly #hashKey: string; #isEnabled = true; readonly #memoryCache = new Map(); - public constructor(cwd: string, packageJsonHash: string, logger: Pail) { + public constructor(cwd: string, cachePath: string | undefined, hashKey: string, logger: Pail) { this.#cwd = cwd; - this.#cachePath = findCacheDirSync("@visulima/packem", { - cwd, - }); - - this.#packageJsonHash = packageJsonHash; + this.#hashKey = hashKey; - if (this.#cachePath === undefined) { - logger.debug("Could not create cache directory."); + if (cachePath === undefined) { + logger.debug({ + message: "Could not create cache directory.", + prefix: "file-cache", + }); } else { - logger.debug(`Cache path is: ${this.#cachePath}`); + this.#cachePath = cachePath; + + logger.debug({ + message: "Cache path is: " + this.#cachePath, + prefix: "file-cache", + }); } + + this.createOrUpdateKeyStorage(hashKey, logger); } public set isEnabled(value: boolean) { this.#isEnabled = value; } - public get cachePath(): string | undefined { - return this.#cachePath; - } - public has(name: string, subDirectory?: string): boolean { if (!this.#isEnabled) { return false; @@ -119,7 +120,36 @@ class FileCache { optimizedName = optimizedName.replaceAll(":", "-"); - return join(this.#cachePath as string, this.#packageJsonHash, subDirectory?.replaceAll(":", "-") ?? "", toNamespacedPath(optimizedName)); + return join(this.#cachePath as string, this.#hashKey, subDirectory?.replaceAll(":", "-") ?? "", toNamespacedPath(optimizedName)); + } + + private createOrUpdateKeyStorage(hashKey: string, logger: Pail): void { + try { + let keyStore: Record = {}; + + const keyStorePath = join(this.#cachePath as string, "keystore.json"); + + if (isAccessibleSync(keyStorePath)) { + keyStore = readJsonSync(keyStorePath); + } + + // eslint-disable-next-line security/detect-object-injection + if (keyStore[hashKey] === undefined) { + // eslint-disable-next-line security/detect-object-injection + keyStore[hashKey] = new Date().toISOString(); + } + + writeJsonSync(keyStorePath, keyStore, { + overwrite: true, + }); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (error: any) { + logger.debug({ + context: error, + message: error.message, + prefix: "file-cache", + }); + } } }